import cliutils, logging, json, os, ospaths, osproc, sequtils, streams,
  strtabs, strutils, tables, tempfile, times, uuids

import ./configuration
import nre except toSeq
from posix import link, realpath
from algorithm import sorted

type
  Workspace = ref object      ## Data needed by internal build process
    buildDataDir*: string     ## absolute path to the global build data directory for this project
    buildRef*: string         ## git-style commit reference to the revision we are building
    dir*: string              ## absolute path to the working directory
    env*: StringTableRef      ## environment variables for all build processes
    logLevel*: Level          ## log level for output messages
    openedFiles*: seq[File]   ## all files that we have opened that need to be closed
    outputHandler*: HandleProcMsgCB ## handler for process output
    project*: ProjectConfig   ## the project configuration
    projectDef*: ProjectDef   ## the StrawBoss project definition
    runRequest*: RunRequest   ## the RunRequest that initated the current build
    status*: BuildStatus      ## the current status of the build
    step*: Step               ## the step we're building
    version*: string          ## project version as returned by versionCmd

  Worker* = object
    runId*: UUID
    projectName*: string
    process*: Process

  NotFoundException* = object of Exception

proc newCopy(w: Workspace): Workspace =
  var newEnv: StringTableRef = newStringTable()
  newEnv[] = w.env[]

  result = Workspace(
    buildDataDir: w.buildDataDir,
    buildRef: w.buildRef,
    dir: w.dir,
    env: newEnv,
    logLevel: w.logLevel,
    # workspaces are only responsible for files they have actually openend
    openedFiles: @[],
    outputHandler: w.outputHandler,
    project: w.project,
    projectDef: w.projectDef,
    runRequest: w.runRequest,
    status: w.status,
    step: w.step,
    version: w.version)

const WKSP_ROOT = "/strawboss/wksp"
const ARTIFACTS_ROOT = "/strawboss/artifacts"

proc execWithOutput(wksp: Workspace, cmd, workingDir: string,
    args: openarray[string], env: StringTableRef,
    options: set[ProcessOption] = {poUsePath},
    msgCB: HandleProcMsgCB = nil):
    tuple[output: TaintedString, error: TaintedString, exitCode: int]
    {.tags: [ExecIOEffect, ReadIOEffect, RootEffect] .} =

  # Look for a container image to use
  let containerImage =
    if wksp.step.containerImage.len > 0: wksp.step.containerImage
    else: wksp.project.containerImage

  if containerImage.len == 0:
    return execWithOutput(cmd, workingDir, args, env, options, msgCB)

  var fullEnv = newStringTable(modeCaseSensitive)
  for k,v in env: fullEnv[k] = v

  var fullArgs = @["run", "-w", WKSP_ROOT, "-v", wksp.dir & ":" & WKSP_ROOT  ]

  if wksp.step.name.len == 0:
    for depStep in wksp.step.depends:
      fullArgs.add(["-v", ARTIFACTS_ROOT / depStep])
      fullEnv[depStep & "_DIR"] = ARTIFACTS_ROOT / depStep

  let envFile = mkstemp().name
  writeFile(envFile, toSeq(fullEnv.pairs()).mapIt(it[0] & "=" & it[1]).join("\n"))
  
  fullArgs.add(["--env-file", envFile])
  fullArgs.add(containerImage)
  fullArgs.add(cmd)

  echo "Executing docker command: \n\t" & "docker " & $(fullArgs & @args)
  return execWithOutput("docker", wksp.dir, fullArgs & @args, fullEnv, options, msgCB)

proc exec(w: Workspace, cmd, workingDir: string, args: openarray[string],
    env: StringTableRef, options: set[ProcessOption] = {poUsePath},
    msgCB: HandleProcMsgCB = nil): int
    {.tags: [ExecIOEffect, ReadIOEffect, RootEffect] .} =

  return execWithOutput(w, cmd, workingDir, args, env, options, msgCB)[2]

# Utility methods for Workspace activities
proc sendStatusMsg(oh: HandleProcMsgCB, status: BuildStatus): void =
  if not oh.isNil:
    oh.sendMsg($status.state & ": " & status.details, "", "strawboss")

proc sendMsg(w: Workspace, msg: TaintedString): void =
  w.outputHandler.sendMsg(msg, "", "strawboss")

proc sendMsg(w: Workspace, l: Level, msg: TaintedString): void =
  if l >= w.logLevel: w.sendMsg(msg)

proc sendErrMsg(w: Workspace, msg: TaintedString): void =
  w.outputHandler.sendMsg("", msg, "strawboss")

proc sendErrMsg(w: Workspace, l: Level, msg: TaintedString): void =
  if l >= w.logLevel: w.sendErrMsg(msg)

proc resolveEnvVars(wksp: Workspace, line: string): string =
  result = line
  for found in line.findAll(re"\$\w+|\$\{[^}]+\}"):
    let key = if found[1] == '{': found[2..^2] else: found[1..^1]
    if wksp.env.hasKey(key): result = result.replace(found, wksp.env[key])
  wksp.sendMsg(lvlDebug, "Variable substitution: \n\t" & line &
    "\n\t" & result)

proc publishStatus(wksp: Workspace, state: BuildState, details: string): void =
  ## Update the status for a Workspace and publish this status to the
  ## Workspace's status file and any output message handlers.
  wksp.status = BuildStatus(
    runId: $wksp.runRequest.runId,
    state: state,
    details: details,
    version: wksp.version)

  # Write to our run directory, and to our version status
  writeFile(wksp.buildDataDir / "runs" /
    $wksp.runRequest.runId & ".status.json", $wksp.status)

  # If we have our step we can save status to the step status
  if wksp.step.name.len > 0:
    let stepStatusDir = wksp.buildDataDir / "status" / wksp.step.name
    if not existsDir(stepStatusDir): createDir(stepStatusDir)
    writeFile(stepStatusDir / wksp.version & ".json", $wksp.status)

    # If we were asked to build a ref that is not the version directly (like
    # "master" or something), then let's also save our status under that name.
    # We're probably overwriting a prior status, but that's OK.
    if wksp.runRequest.buildRef != wksp.version:
      writeFile(stepStatusDir / wksp.runRequest.buildRef & ".json",
                $wksp.status)

  wksp.outputHandler.sendStatusMsg(wksp.status)

proc ensureProjectDirsExist(cfg: StrawBossConfig, p: ProjectDef): void =
  for subdir in ["configurations", "runs", "status", "artifacts"]:
    let fullPath = cfg.buildDataDir / p.name / subdir
    if not existsDir(fullPath):
      createDir(fullPath)

# Data and configuration access

proc getProject*(cfg: StrawBossConfig, projectName: string): ProjectDef =
  ## Get a project definition by name from the service configuration
  let candidates = cfg.projects.filterIt(it.name == projectName)
  if candidates.len == 0:
    raise newException(NotFoundException, "no project named " & projectName)
  elif candidates.len > 1:
    raise newException(NotFoundException, "multiple projects named " & projectName)
  else: result = candidates[0]

proc setProject*(cfg: var StrawBossConfig, projectName: string, newDef: ProjectDef): void =
  ## Add a project definition to the service configuration
  var found = false
  for idx in 0..<cfg.projects.len:
    if cfg.projects[idx].name == projectName:
      cfg.projects[idx] = newDef
      found = true
      break

  if not found: cfg.projects.add(newDef)

proc listVersions*(cfg: StrawBossConfig, projectName: string): seq[string] =
  ## List the versions that have been built for a project.

  let project = cfg.getProject(projectName)

  ensureProjectDirsExist(cfg, project)

  let versionFiles = filesMatching(
    cfg.buildDataDir / project.name / "configurations/*.json")

  result = versionFiles.map(proc(s: string): string =
    let slashIdx = s.rfind('/')
    result = s[(slashIdx + 1)..^6])

proc getBuildStatus*(cfg: StrawBossConfig,
                     projectName, stepName, buildRef: string): BuildStatus =

  let project = cfg.getProject(projectName)

  let statusFile = cfg.buildDataDir / project.name / "status" /
    stepName / buildRef & ".json"

  if not existsFile(statusFile):
    raise newException(NotFoundException,
      stepName & " has never been built for " & projectName & "@" & buildRef)

  result = loadBuildStatus(statusFile)


proc listArtifacts*(cfg: StrawBossConfig,
                    projectName, stepName, version: string): seq[string] =
  ## List the artifacts that have been built for a step.

  let project = cfg.getProject(projectName)

  ensureProjectDirsExist(cfg, project)

  let buildStatus = cfg.getBuildStatus(projectName, stepName, version)

  if buildStatus.state != BuildState.complete:
    raise newException(NotFoundException, "step " & stepName &
      " has never been successfully built for " & projectName & "@" & version)

  result = filesMatching(
      cfg.buildDataDir / project.name / "artifacts" / stepName / version / "*")
    .mapIt(it.extractFilename)

proc getArtifactPath*(cfg: StrawBossConfig,
  projectName, stepName, version, artifactName: string): string =

  let artifacts = cfg.listArtifacts(projectName, stepName, version)
  if not artifacts.contains(artifactName):
    raise newException(NotFoundException, "no artifact named " &
      artifactName & " exists for step " & stepName & " in project " &
      projectName & "@" & version)

  result = cfg.buildDataDir / projectName / "artifacts" / stepName / version / artifactName

proc existsRun*(cfg: StrawBossConfig, projectName, runId: string): bool =
  existsFile(cfg.buildDataDir / projectName / "runs" / runId & ".request.json")

proc getRun*(cfg: StrawBossConfig, projectName, runId: string): Run =
  let project = cfg.getProject(projectName)
  let runsPath = cfg.buildDataDir / project.name / "runs"

  try: result = Run(
    id: parseUUID(runId),
    request: loadRunRequest(runsPath / runId & ".request.json"),
    status: loadBuildStatus(runsPath / runId & ".status.json"))
  except: raiseEx "unable to load run information for id " & runId

proc listRuns*(cfg: StrawBossConfig, projectName: string): seq[Run] =
  ## List the runs that have been performed for a project.
  let project = cfg.getProject(projectName)
  ensureProjectDirsExist(cfg, project)

  let runsPath = cfg.buildDataDir / project.name / "runs"
  let reqPaths = filesMatching(runsPath / "*.request.json")

  result = reqPaths.map(proc(reqPath: string): Run =
    let runId = reqPath[(runsPath.len + 1)..^14]
    result = Run(
      id: parseUUID(runId),
      request: loadRunRequest(reqPath),
      status: loadBuildStatus(runsPath / runId & ".status.json")))

proc getLogs*(cfg: StrawBossConfig, projectname, runId: string): RunLogs =
  let project = cfg.getProject(projectName)
  let runsPath = cfg.buildDataDir / project.name / "runs"

  try: result = RunLogs(
    runId: parseUUID(runId),
    stdout: toSeq(lines(runsPath / runId & ".stdout.log")),
    stderr: toSeq(lines(runsPath / runId & ".stderr.log")))
  except: raiseEx "unable to load logs for run " & runId

proc getProjectConfig*(cfg: StrawBossConfig,
                       projectName, version: string): ProjectConfig =

  let project = cfg.getProject(projectName)
  ensureProjectDirsExist(cfg, project)

  # If they didn't give us a version, let try to figure out what is the latest one.
  var confFilePath: string

  if version.len == 0:

    let candidatePaths = filesMatching(
      cfg.buildDataDir / project.name / "configurations/*.json")

    if candidatePaths.len == 0:
      raise newException(NotFoundException,
        "no versions of this project have been built")

    let modTimes = candidatePaths.mapIt(it.getLastModificationTime)
    confFilePath = sorted(zip(candidatePaths, modTimes),
                              proc(a, b: tuple): int = cmp(a.b, b.b))[0].a

    #cachedFilePath = sorted(zip(confFilePaths, modTimes),
    #                        proc (a, b: tuple): int = cmp(a.b, b.b))[0].a

  # If they did, let's try to load that
  else:
    confFilePath =
      cfg.buildDataDir / project.name / "configurations" / version & ".json"

  if not existsFile(confFilePath):
    raise newException(NotFoundException,
      projectName & " version " & version & " has never been built")

  result = loadProjectConfig(confFilePath)


# Internal working methods.
proc setupProject(wksp: Workspace) =

  wksp.sendMsg(lvlDebug, "Setting up project.")

  # Clone the project into the $temp directory
  let cloneArgs = @["clone", wksp.projectDef.repo, wksp.dir]
  wksp.sendMsg(lvlDebug, "git " & $cloneArgs)

  let cloneResult = exec("git", ".", cloneArgs, wksp.env, {poUsePath},
    wksp.outputHandler)

  if cloneResult != 0:
    raiseEx "unable to clone repo for '" & wksp.projectDef.name & "'"

  # Checkout the requested ref
  let checkoutArgs = @["checkout", wksp.buildRef]
  wksp.sendMsg(lvlDebug, "git " & $checkoutArgs)

  let checkoutResult = exec("git", wksp.dir, checkoutArgs,
    wksp.env, {poUsePath}, wksp.outputHandler)

  if checkoutResult != 0:
    raiseEx "unable to checkout ref " & wksp.buildRef &
            " for '" & wksp.projectDef.name & "'"

  # Find the strawboss project configuration
  let projCfgFile = wksp.dir / wksp.projectDef.cfgFilePath
  wksp.sendMsg(lvlDebug, "Looking for project configuration at '" & projCfgFile & "'")
  if not existsFile(projCfgFile):
    raiseEx "Cannot find strawboss project configuration in the project " &
      "repo (expected at '" & wksp.projectDef.cfgFilePath & "')."

  wksp.project = loadProjectConfig(projCfgFile)

  # Merge in the project-defined env vars
  for k, v in wksp.projectDef.envVars: wksp.env[k] = v

  # Get the build version
  let versionResult = execWithOutput(
    wksp.project.versionCmd,    # command
    wksp.dir,                   # working dir
    [],                         # args
    wksp.env,                   # environment
    {poUsePath, poEvalCommand}) # options

  if versionResult.exitCode != 0:
    raiseEx "Version command (" & wksp.project.versionCmd &
            ") returned non-zero exit code."

  wksp.version = versionResult.output.strip
  wksp.env["VERSION"] = wksp.version

proc doStep*(wksp: Workspace, step: Step): BuildStatus =

  ## Lower-level method to execute a given step within the context of a project
  ## workspace that is setup and configured. May be called recursively to
  ## satisfy step dependencies.

  wksp.step = step

  let artifactsDir = wksp.buildDataDir / "artifacts" / step.name / wksp.version

  if not existsDir(artifactsDir): createDir(artifactsDir)

  # Have we tried to build this before and are we caching the results?
  let statusFilePath = wksp.buildDataDir / "status" / step.name /
    wksp.version & ".json"

  if existsFile(statusFilePath) and not step.dontSkip:
    let prevStatus = loadBuildStatus(statusFilePath)

    # If we succeeded last time, no need to rebuild
    if prevStatus.state == BuildState.complete:
      wksp.publishStatus(BuildState.stepComplete,
        "Skipping step '" & step.name & "' for version '" & wksp.version &
          "': already completed.")
      return wksp.status
    else:
      wksp.sendMsg(
        "Rebuilding failed step '" & step.name & "' for version '" &
        wksp.version & "'.")

  let SB_EXPECTED_VARS = ["VERSION"]

  wksp.publishStatus(BuildState.running,
    "running '" & step.name & "' for version " & wksp.version &
    " from " & wksp.buildRef)

  # Ensure all expected environment variables are present.
  for k in (step.expectedEnv & @SB_EXPECTED_VARS):
    if not wksp.env.hasKey(k):
      raiseEx "step " & step.name & " failed: missing required env variable: " & k

  # Ensure that artifacts in steps we depend on are present
  # TODO: detect circular-references in dependency trees.
  for dep in step.depends:
    if not wksp.project.steps.hasKey(dep):
      raiseEx step.name & " depends on " & dep &
              " but there is no step named " & dep
    let depStep = wksp.project.steps[dep]

    # Run that step (may get skipped)
    let runStatus = doStep(core.newCopy(wksp), depStep)

    if not (runStatus.state == BuildState.stepComplete):
      raiseEx "dependent step failed: " & depStep.name

    wksp.sendMsg(lvlDebug, "dependent step '" & depStep.name &
      "'completed, resuming '" & wksp.step.name & "'")

    # Add the artifacts directory for the dependent step to our env so that
    # further steps can reference it via $<stepname>_DIR
    wksp.env[depStep.name & "_DIR"] = wksp.buildDataDir / "artifacts" /
      dep / wksp.version

  # Run the step command, piping in cmdInput
  let stepCmd = wksp.resolveEnvVars(step.stepCmd)
  let cmdName = if stepCmd.rfind("/") >= 0: stepCmd[(stepCmd.rfind("/") + 1)..^1]
                else: stepCmd
  wksp.sendMsg step.name & ": starting stepCmd: " & stepCmd
  let cmdProc = startProcess(stepCmd,
    wksp.dir / step.workingDir, [], wksp.env, {poUsePath, poEvalCommand})

  let cmdInStream = inputStream(cmdProc)

  # Replace env variables in step cmdInput as we pipe it in
  for line in step.cmdInput: cmdInStream.writeLine(wksp.resolveEnvVars(line))
  cmdInStream.flush()
  cmdInStream.close()

  let cmdResult = waitFor(cmdProc, wksp.outputHandler, cmdName)

  if cmdResult != 0:
    raiseEx "step " & step.name & " failed: step command returned non-zero exit code"

  # Gather the output artifacts (if we have any)
  wksp.sendMsg "artifacts: " & $step.artifacts
  if step.artifacts.len > 0:
    for a in step.artifacts:
      let artifactPath = wksp.resolveEnvVars(a)
      let artifactName = artifactPath[(artifactPath.rfind("/")+1)..^1]
      try:
        wksp.sendMsg "copy " &
          wksp.dir / step.workingDir / artifactPath & " -> " &
          artifactsDir / artifactName

        copyFileWithPermissions(wksp.dir / step.workingDir / artifactPath,
                                artifactsDir / artifactName)
      except:
        raiseEx "step " & step.name & " failed: unable to copy artifact " &
          artifactPath & ":\n" & getCurrentExceptionMsg()

  wksp.publishStatus(BuildState.stepComplete, "step " & step.name & " complete")
  result = wksp.status

proc run*(cfg: StrawBossConfig, req: RunRequest,
              outputHandler: HandleProcMsgCB = nil): BuildStatus =

  ## Execute a RunReuest given the StrawBoss configuration. This is the main
  ## entrypoint to  running a build step.

  result = BuildStatus(
    runId: $req.runId,
    state: BuildState.setup,
    details: "initializing build workspace",
    version: "")
  outputHandler.sendStatusMsg(result)

  var wksp: Workspace

  try:
    # Find the project definition
    let projectDef = cfg.getProject(req.projectName)

    # Make sure the build data directories for this project exist.
    ensureProjectDirsExist(cfg, projectDef)

    # Update our run status
    let runDir = cfg.buildDataDir / projectDef.name / "runs"
    writeFile(runDir / $req.runId & ".status.json", $result)

    # Read in the existing system environment
    var env = loadEnv()
    env["GIT_DIR"] = ".git"

    # Make sure we have a workspace directory
    assert req.workspaceDir.isAbsolute
    if not existsDir(req.workspaceDir): createDir(req.workspaceDir)

    # Setup our STDOUT and STDERR files
    let stdoutFile = open(runDir / $req.runId & ".stdout.log", fmWrite)
    let stderrFile = open(runDir / $req.runId & ".stderr.log", fmWrite)

    let logFilesOH = makeProcMsgHandler(stdoutFile, stderrFile)

    wksp = Workspace(
      buildDataDir: cfg.buildDataDir / projectDef.name,
      buildRef:
        if req.buildRef.len > 0: req.buildRef
        else: projectDef.defaultBranch,
      dir: req.workspaceDir,
      env: env,
      logLevel: cfg.logLevel,
      openedFiles: @[stdoutFile, stderrFile],
      outputHandler: combineProcMsgHandlers(outputHandler, logFilesOH),
      project: ProjectConfig(),
      projectDef: projectDef,
      runRequest: req,
      status: result,
      step: Step(),
      version: "")

  except:
    when not defined(release): echo getCurrentException().getStackTrace()
    result = BuildStatus(runId: $req.runId, state: BuildState.failed,
                         details: getCurrentExceptionMsg(), version: "")
    try: outputHandler.sendStatusMsg(result)
    except: discard ""
    return

  try:
    # Clone the repo and setup the working environment
    wksp.publishStatus(BuildState.setup,
      "cloning project repo and preparing to run '" & req.stepName & "'")
    wksp.setupProject()

    # Update our cache of project configurations.
    # TODO: what happens if this fails?
    copyFileWithPermissions(
      wksp.dir / wksp.projectDef.cfgFilePath,
      wksp.buildDataDir / "configurations" / wksp.version & ".json")

    # Find the requested step
    if not wksp.project.steps.hasKey(req.stepName):
      raiseEx "no step name '" & req.stepName & "' for " & req.projectName
    var step = wksp.project.steps[req.stepName]

    if req.forceRebuild: step.dontSkip = true

    var buildStatus = doStep(wksp, step)
    if buildStatus.state == BuildState.stepComplete:
      buildStatus.state = BuildState.complete
      wksp.publishStatus(buildStatus.state, "all steps complete")

    result = wksp.status

  except:
    when not defined(release): echo getCurrentException().getStackTrace()
    let msg = getCurrentExceptionMsg()
    try:
      wksp.publishStatus(BuildState.failed, msg)
      result = wksp.status
    except:
      result = BuildStatus(runId: $req.runId, state: BuildState.failed,
                           details: msg, version: "")
      try: outputHandler.sendStatusMsg(result)
      except: discard ""

  finally:
    if wksp != nil:
      # Close open files
      for f in wksp.openedFiles:
        try: close(f)
        except: discard ""

proc spawnWorker*(cfg: StrawBossConfig, req: RunRequest):
  tuple[status: BuildStatus, worker: Worker] =

  # Find the project definition (will throw appropriate exceptions)
  let projectDef = cfg.getProject(req.projectName)
  let runDir = cfg.buildDataDir / projectDef.name / "runs"
  let reqFile = runDir / $req.runId & ".request.json"
  let statusFile = runDir / $req.runId & ".status.json"

  try:
    # Make sure the build data directories for this project exist.
    ensureProjectDirsExist(cfg, projectDef)

    # Save the run request
    writeFile(reqFile, $req)

    # Write the initial build status (queued).
    let queuedStatus = BuildStatus(
      runId: $req.runId,
      state: BuildState.queued,
      details: "request queued for execution",
      version: "")
    writeFile(statusFile, $queuedStatus)

    var args = @["run", reqFile, "-c", cfg.filePath]
    debug "Launching worker: " & cfg.pathToExe & " " & args.join(" ")

    let worker = Worker(
      runId: req.runId,
      projectName: projectDef.name,
      process: startProcess(cfg.pathToExe, ".", args, loadEnv(), {poUsePath}))

    result = (queuedStatus, worker)

  except:
    let exMsg = "run request rejected: " & getCurrentExceptionMsg()
    try:
      writeFile(statusFile,
        $(BuildStatus(runId: $req.runId, state: BuildState.rejected,
                      details: exMsg, version: "")))
    except: discard ""
    raiseEx exMsg