From 923ec24d27fa589cb6544d592d4444f34cffe38d Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 28 May 2021 10:31:19 -0700
Subject: [PATCH 01/23] add tools/digger

---
 tools/digger.nim | 227 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 227 insertions(+)
 create mode 100644 tools/digger.nim

diff --git a/tools/digger.nim b/tools/digger.nim
new file mode 100644
index 0000000000000..a5f1102b6e552
--- /dev/null
+++ b/tools/digger.nim
@@ -0,0 +1,227 @@
+#[
+
+## notes
+can build as far back as: v0.12.0~157
+]#
+
+import std/[os, osproc, strformat, macros, strutils, tables, algorithm]
+import timn/dbgs
+
+type
+  CsourcesOpt = ref object
+    url: string
+    dir: string
+    rev: string
+    binDir: string
+    csourcesBuildArgs: string
+    revs: seq[string]
+    fetch: bool
+    name: string
+    nimCsourcesExe: string
+  DiggerOpt = object
+    # input fields
+    rev: string
+    nimDir: string
+    compileNim: bool
+    fetch: bool
+    csourcesBuildArgs: string
+    buildAllCsources: bool
+    verbose: bool
+
+    # bisect cmds
+    # TODO: specify whether we should compile nim
+    bisectCmd: string
+    bisectBugfix: bool
+    oldrev, newrev: string
+
+    # state fieles (TODO: split out these)
+    coptv0: CsourcesOpt
+    coptv1: CsourcesOpt
+    binDir: string
+
+const
+  nimUrl = "https://github.com/nim-lang/Nim"
+  csourcesUrl = "https://github.com/nim-lang/csources.git"
+  csourcesV1Url = "https://github.com/nim-lang/csources_v1.git"
+  csourcesName = "csources"
+  csourcesV1Name = "csources_v1"
+  csourcesRevs = "v0.9.4 v0.13.0 v0.15.2 v0.16.0 v0.17.0 v0.17.2 v0.18.0 v0.19.0 v0.20.0 64e3477".split
+  csourcesV1Revs = "a8a5241f9475099c823cfe1a5e0ca4022ac201ff".split
+  NimDiggerEnv = "NIMDIGGER_HOME"
+
+var verbose = false
+proc isSimulate(): bool =
+  defined(nimDiggerSimulate)
+
+proc runCmd(cmd: string) =
+  # TODO: allow `dir` param
+  if isSimulate():
+    dbg cmd
+  else:
+    if verbose: dbg cmd
+    doAssert execShellCmd(cmd) == 0, cmd
+
+proc runCmdOutput(cmd: string, dir = ""): string =
+  if verbose: dbg cmd, dir
+  let (outp, status) = execCmdEx(cmd, workingDir = dir)
+  doAssert status == 0, &"status: {status}\ncmd: {cmd}\ndir: {dir}\noutput: {outp}"
+  result = outp
+  stripLineEnd(result)
+
+macro ctor(obj: untyped, a: varargs[untyped]): untyped =
+  ## Generates an object constructor call from a list of fields.
+  # xxx expose in some `fusion/macros` or std/macros; FACTOR with pr_fusion_globs PR
+  runnableExamples:
+    type Foo = object
+      a, b: int
+    doAssert Foo.ctor(a,b) == Foo(a: a, b: b)
+  result = nnkObjConstr.newTree(obj)
+  for ai in a: result.add nnkExprColonExpr.newTree(ai, ai)
+
+proc gitClone(url: string, dir: string) = runCmd fmt"git clone -q {url} {dir.quoteShell}"
+proc gitResetHard(dir: string, rev: string) = runCmd fmt"git -C {dir.quoteShell} reset --hard {rev}"
+proc gitFetch(dir: string) = runCmd fmt"git -C {dir.quoteShell} fetch"
+proc gitLatestTag(dir: string): string = runCmdOutput("git describe --abbrev=0 HEAD", dir)
+proc gitCheck(dir: string) =
+  # checks whether we're in a valid git repo; there may be better ways
+  discard runCmdOutput("git describe HEAD", dir)
+
+proc gitIsAncestorOf(dir: string, rev1, rev2: string): bool =
+  gitCheck(dir)
+  execShellCmd(fmt"git -C {dir.quoteShell} merge-base --is-ancestor {rev1} {rev2}") == 0
+
+proc isGitNimTag(tag: string): bool =
+  if not tag.startsWith "v":
+    return false
+  let ver = tag[1..^1].split(".")
+  return ver.len == 3
+
+proc parseNimGitTag(tag: string): (int, int, int) =
+  doAssert tag.isGitNimTag, tag
+  let ver = tag[1..^1].split(".")
+  template impl(i) =
+    # improve pending https://github.com/nim-lang/Nim/pull/18038
+    result[i] = ver[i].parseInt
+  impl 0
+  impl 1
+  impl 2
+
+proc toNimCsourcesExe(binDir: string, name: string, rev: string): string =
+  let rev2 = rev.replace(".", "_")
+  result = binDir / fmt"nim_digger_{name}_{rev2}"
+
+proc buildCsourcesRev(copt: CsourcesOpt) =
+  # sync with `_nimBuildCsourcesIfNeeded`
+  let csourcesExe = toNimCsourcesExe(copt.binDir, copt.name, copt.rev)
+  if csourcesExe.fileExists:
+    return
+  if verbose: dbg copt
+  if not copt.dir.dirExists: gitClone(copt.url, copt.dir)
+  if copt.fetch: gitFetch(copt.dir)
+  gitResetHard(copt.dir, copt.rev)
+  when defined(bsd):
+    let make = "gmake"
+  else:
+    let make = "make"
+  let ncpu = countProcessors()
+  if copt.rev.isGitNimTag and copt.rev.parseNimGitTag < (0,15,2):
+    # avoids: make: *** No rule to make target `c_code/3_2/compiler_testability.o', needed by `../bin/nim'.  Stop.
+    discard runCmdOutput(fmt"sh build.sh {copt.csourcesBuildArgs}", copt.dir)
+  else:
+    discard runCmdOutput(fmt"{make} -j {ncpu + 2} -l {ncpu} {copt.csourcesBuildArgs}", copt.dir)
+  if isSimulate():
+    dbg csourcesExe
+  else:
+    copyFile(copt.binDir / "nim", csourcesExe) # TODO: windows exe etc?
+
+proc buildCsourcesAnyRevs(copt: CsourcesOpt) =
+  for rev in copt.revs:
+    copt.rev = rev
+    buildCsourcesRev(copt)
+
+proc parseKeyVal(a: string): OrderedTable[string, string] =
+  ## parse bash-like entries of the form key=val
+  let a2 = a.splitLines
+  for i, ai in a2:
+    if ai.len == 0 or ai.startsWith "#": continue
+    let b = split(ai, "=", maxsplit = 1)
+    doAssert b.len == 2, $(ai, b)
+    result[b[0]] = b[1]
+
+proc toCsourcesRev(rev: string): string =
+  let ver = rev.parseNimGitTag
+  if ver >= (1, 0, 0): return csourcesRevs[^1]
+  for a in csourcesRevs[1 ..< ^1].reversed:
+    if ver >= a.parseNimGitTag: return a
+  # v0.9.4 seems broken
+  return csourcesRevs[1]
+
+proc getNimCsourcesAnyExe(opt: DiggerOpt): CsourcesOpt =
+  let file = opt.nimDir/"config/build_config.txt" # for newer nim versions, this file specifies correct csources_v1 to use
+  if file.fileExists:
+    let tab = file.readFile.parseKeyVal
+    result = opt.coptv1
+    result.rev = tab["nim_csourcesHash"]
+  elif gitIsAncestorOf(opt.nimDir, "a9b62de", opt.rev): # commit that introduced csources_v1
+    result = opt.coptv1
+    result.rev = csourcesV1Revs[0]
+  else:
+    let tag = gitLatestTag(opt.nimDir)
+    result = opt.coptv0
+    result.rev = tag.toCsourcesRev
+  result.nimCsourcesExe = toNimCsourcesExe(opt.binDir, result.name, result.rev)
+
+proc main2(opt: DiggerOpt) =
+  var opt = opt
+  let nimdiggerHome = getEnv(NimDiggerEnv, getHomeDir() / ".nimdigger")
+  if opt.nimDir.len == 0:
+    opt.nimDir = nimdiggerHome / "cache/Nim"
+  if verbose: dbg opt
+  let nimDir = opt.nimDir
+  let csourcesDir = nimDir/csourcesName
+  let csourcesV1Dir = nimDir/csourcesV1Name
+  opt.binDir = nimDir/"bin"
+  let nimDiggerExe = opt.binDir / "nim_digger"
+
+  if nimDir.dirExists:
+    doAssert fileExists(nimDir / "lib/system.nim"), fmt"nimDir is not a nim repo: {nimDir}"
+  else:
+    createDir nimDir.parentDir
+    gitClone(nimUrl, nimDir)
+
+  opt.coptv0 = CsourcesOpt(dir: csourcesDir, url: csourcesUrl, name: csourcesName, revs: csourcesRevs)
+  opt.coptv1 = CsourcesOpt(dir: csourcesV1Dir, url: csourcesV1Url, name: csourcesV1Name, revs: csourcesV1Revs)
+  for copt in [opt.coptv0, opt.coptv1]:
+    copt.binDir = opt.binDir
+    copt.fetch = opt.fetch
+    if opt.buildAllCsources:
+      buildCsourcesAnyRevs(copt)
+
+  if opt.fetch: gitFetch(nimDir)
+  if opt.rev.len > 0: gitResetHard(nimDir, opt.rev)
+  else: opt.rev = "HEAD"
+
+  if opt.compileNim:
+    let copt = getNimCsourcesAnyExe(opt)
+    buildCsourcesRev(copt)
+    # TODO: we could also cache those, optionally maybe (could get large?) or use this as hint in nim bisect to prefer those
+    discard runCmdOutput(fmt"{copt.nimCsourcesExe} c -o:{nimDiggerExe} --hints:off --skipUserCfg compiler/nim.nim", nimDir)
+
+  if opt.bisectCmd.len > 0:
+    doAssert opt.oldrev.len > 0 # for regressions, aka goodrev
+    doAssert opt.newrev.len > 0 # for a regressions, aka badrev 
+    runCmd(fmt"git -C {opt.nimDir.quoteShell} bisect start {opt.newrev} {opt.oldrev}")
+    let exe = getAppFileName()
+    var msg2 = opt.bisectCmd
+    if opt.bisectBugfix:
+      msg2 = fmt"! ({msg2})" # negate exit code
+    let bisectCmd2 = fmt"{exe} --compileNim && cp {nimDiggerExe.quoteShell} bin/nim && {msg2}"
+    runCmd(fmt"git -C {opt.nimDir.quoteShell} bisect run bash -c {bisectCmd2.quoteShell}")
+
+proc main(rev = "", nimDir = "", compileNim = false, fetch = false, bisectCmd = "", newrev = "", oldrev = "", bisectBugfix = false, verbose = false) =
+  digger.verbose = verbose
+  main2(DiggerOpt.ctor(rev, nimDir, compileNim, fetch, bisectCmd, newrev, oldrev, bisectBugfix))
+
+when isMainModule:
+  import pkg/cligen
+  dispatch main

From c119e0e8955f5b8b7a743dda6964f5c398129f5f Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 28 May 2021 11:41:13 -0700
Subject: [PATCH 02/23] bugfix

---
 tools/{digger.nim => nimdigger.nim} | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)
 rename tools/{digger.nim => nimdigger.nim} (93%)

diff --git a/tools/digger.nim b/tools/nimdigger.nim
similarity index 93%
rename from tools/digger.nim
rename to tools/nimdigger.nim
index a5f1102b6e552..dc0d20f2e14cc 100644
--- a/tools/digger.nim
+++ b/tools/nimdigger.nim
@@ -50,6 +50,7 @@ const
   NimDiggerEnv = "NIMDIGGER_HOME"
 
 var verbose = false
+
 proc isSimulate(): bool =
   defined(nimDiggerSimulate)
 
@@ -64,7 +65,7 @@ proc runCmd(cmd: string) =
 proc runCmdOutput(cmd: string, dir = ""): string =
   if verbose: dbg cmd, dir
   let (outp, status) = execCmdEx(cmd, workingDir = dir)
-  doAssert status == 0, &"status: {status}\ncmd: {cmd}\ndir: {dir}\noutput: {outp}"
+  doAssert status == 0, indent(&"status: {status}\ncmd: {cmd}\ndir: {dir}\noutput: {outp}", 2)
   result = outp
   stripLineEnd(result)
 
@@ -108,7 +109,7 @@ proc parseNimGitTag(tag: string): (int, int, int) =
 
 proc toNimCsourcesExe(binDir: string, name: string, rev: string): string =
   let rev2 = rev.replace(".", "_")
-  result = binDir / fmt"nim_digger_{name}_{rev2}"
+  result = binDir / fmt"nim_nimdigger_{name}_{rev2}"
 
 proc buildCsourcesRev(copt: CsourcesOpt) =
   # sync with `_nimBuildCsourcesIfNeeded`
@@ -123,6 +124,8 @@ proc buildCsourcesRev(copt: CsourcesOpt) =
     let make = "gmake"
   else:
     let make = "make"
+  let oldNim = copt.binDir / "nim"
+  removeFile(oldNim) # otherwise `make` may incorrectly decide there's notthing to build
   let ncpu = countProcessors()
   if copt.rev.isGitNimTag and copt.rev.parseNimGitTag < (0,15,2):
     # avoids: make: *** No rule to make target `c_code/3_2/compiler_testability.o', needed by `../bin/nim'.  Stop.
@@ -132,7 +135,7 @@ proc buildCsourcesRev(copt: CsourcesOpt) =
   if isSimulate():
     dbg csourcesExe
   else:
-    copyFile(copt.binDir / "nim", csourcesExe) # TODO: windows exe etc?
+    copyFile(oldNim, csourcesExe) # TODO: windows: do i need to add exe or it's smart enough?
 
 proc buildCsourcesAnyRevs(copt: CsourcesOpt) =
   for rev in copt.revs:
@@ -181,7 +184,7 @@ proc main2(opt: DiggerOpt) =
   let csourcesDir = nimDir/csourcesName
   let csourcesV1Dir = nimDir/csourcesV1Name
   opt.binDir = nimDir/"bin"
-  let nimDiggerExe = opt.binDir / "nim_digger"
+  let nimDiggerExe = opt.binDir / "nim_nimdigger"
 
   if nimDir.dirExists:
     doAssert fileExists(nimDir / "lib/system.nim"), fmt"nimDir is not a nim repo: {nimDir}"
@@ -215,11 +218,11 @@ proc main2(opt: DiggerOpt) =
     var msg2 = opt.bisectCmd
     if opt.bisectBugfix:
       msg2 = fmt"! ({msg2})" # negate exit code
-    let bisectCmd2 = fmt"{exe} --compileNim && cp {nimDiggerExe.quoteShell} bin/nim && {msg2}"
+    let bisectCmd2 = fmt"{exe} --compileNim && cp {nimDiggerExe.quoteShell} bin/nim && {msg2}" # TODO: inside () in case it does weird things?
     runCmd(fmt"git -C {opt.nimDir.quoteShell} bisect run bash -c {bisectCmd2.quoteShell}")
 
 proc main(rev = "", nimDir = "", compileNim = false, fetch = false, bisectCmd = "", newrev = "", oldrev = "", bisectBugfix = false, verbose = false) =
-  digger.verbose = verbose
+  nimdigger.verbose = verbose
   main2(DiggerOpt.ctor(rev, nimDir, compileNim, fetch, bisectCmd, newrev, oldrev, bisectBugfix))
 
 when isMainModule:

From 137a5d0aab9793002b3c3b63c6078f5870964817 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 28 May 2021 11:47:42 -0700
Subject: [PATCH 03/23] cleanup

---
 tools/nimdigger.nim | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
index dc0d20f2e14cc..14d3f41d563ae 100644
--- a/tools/nimdigger.nim
+++ b/tools/nimdigger.nim
@@ -18,6 +18,7 @@ type
     fetch: bool
     name: string
     nimCsourcesExe: string
+  # DiggerState = ref object
   DiggerOpt = object
     # input fields
     rev: string
@@ -40,11 +41,6 @@ type
     binDir: string
 
 const
-  nimUrl = "https://github.com/nim-lang/Nim"
-  csourcesUrl = "https://github.com/nim-lang/csources.git"
-  csourcesV1Url = "https://github.com/nim-lang/csources_v1.git"
-  csourcesName = "csources"
-  csourcesV1Name = "csources_v1"
   csourcesRevs = "v0.9.4 v0.13.0 v0.15.2 v0.16.0 v0.17.0 v0.17.2 v0.18.0 v0.19.0 v0.20.0 64e3477".split
   csourcesV1Revs = "a8a5241f9475099c823cfe1a5e0ca4022ac201ff".split
   NimDiggerEnv = "NIMDIGGER_HOME"
@@ -175,14 +171,15 @@ proc getNimCsourcesAnyExe(opt: DiggerOpt): CsourcesOpt =
   result.nimCsourcesExe = toNimCsourcesExe(opt.binDir, result.name, result.rev)
 
 proc main2(opt: DiggerOpt) =
+  const
+    csourcesName = "csources"
+    csourcesV1Name = "csources_v1"
   var opt = opt
   let nimdiggerHome = getEnv(NimDiggerEnv, getHomeDir() / ".nimdigger")
   if opt.nimDir.len == 0:
     opt.nimDir = nimdiggerHome / "cache/Nim"
   if verbose: dbg opt
   let nimDir = opt.nimDir
-  let csourcesDir = nimDir/csourcesName
-  let csourcesV1Dir = nimDir/csourcesV1Name
   opt.binDir = nimDir/"bin"
   let nimDiggerExe = opt.binDir / "nim_nimdigger"
 
@@ -190,10 +187,10 @@ proc main2(opt: DiggerOpt) =
     doAssert fileExists(nimDir / "lib/system.nim"), fmt"nimDir is not a nim repo: {nimDir}"
   else:
     createDir nimDir.parentDir
-    gitClone(nimUrl, nimDir)
+    gitClone("https://github.com/nim-lang/Nim", nimDir)
 
-  opt.coptv0 = CsourcesOpt(dir: csourcesDir, url: csourcesUrl, name: csourcesName, revs: csourcesRevs)
-  opt.coptv1 = CsourcesOpt(dir: csourcesV1Dir, url: csourcesV1Url, name: csourcesV1Name, revs: csourcesV1Revs)
+  opt.coptv0 = CsourcesOpt(dir: nimDir/csourcesName, url: "https://github.com/nim-lang/csources.git", name: csourcesName, revs: csourcesRevs)
+  opt.coptv1 = CsourcesOpt(dir: nimDir/csourcesV1Name, url: "https://github.com/nim-lang/csources_v1.git", name: csourcesV1Name, revs: csourcesV1Revs)
   for copt in [opt.coptv0, opt.coptv1]:
     copt.binDir = opt.binDir
     copt.fetch = opt.fetch

From 7a20ac30b5c711e11802d399ad45241f26c460a2 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 28 May 2021 11:59:59 -0700
Subject: [PATCH 04/23] DiggerState

---
 tools/nimdigger.nim | 77 ++++++++++++++++++++++-----------------------
 1 file changed, 38 insertions(+), 39 deletions(-)

diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
index 14d3f41d563ae..e3abda9d9efb1 100644
--- a/tools/nimdigger.nim
+++ b/tools/nimdigger.nim
@@ -8,19 +8,7 @@ import std/[os, osproc, strformat, macros, strutils, tables, algorithm]
 import timn/dbgs
 
 type
-  CsourcesOpt = ref object
-    url: string
-    dir: string
-    rev: string
-    binDir: string
-    csourcesBuildArgs: string
-    revs: seq[string]
-    fetch: bool
-    name: string
-    nimCsourcesExe: string
-  # DiggerState = ref object
-  DiggerOpt = object
-    # input fields
+  DiggerOpt = object ## nimdigger input
     rev: string
     nimDir: string
     compileNim: bool
@@ -34,11 +22,22 @@ type
     bisectCmd: string
     bisectBugfix: bool
     oldrev, newrev: string
-
-    # state fieles (TODO: split out these)
+  CsourcesOpt = ref object
+    url: string
+    dir: string
+    rev: string
+    binDir: string
+    csourcesBuildArgs: string
+    revs: seq[string]
+    fetch: bool
+    name: string
+    nimCsourcesExe: string
+  DiggerState = ref object ## nimdigger internal state
     coptv0: CsourcesOpt
     coptv1: CsourcesOpt
     binDir: string
+    nimDir: string
+    rev: string
 
 const
   csourcesRevs = "v0.9.4 v0.13.0 v0.15.2 v0.16.0 v0.17.0 v0.17.2 v0.18.0 v0.19.0 v0.20.0 64e3477".split
@@ -155,33 +154,33 @@ proc toCsourcesRev(rev: string): string =
   # v0.9.4 seems broken
   return csourcesRevs[1]
 
-proc getNimCsourcesAnyExe(opt: DiggerOpt): CsourcesOpt =
-  let file = opt.nimDir/"config/build_config.txt" # for newer nim versions, this file specifies correct csources_v1 to use
+proc getNimCsourcesAnyExe(state: DiggerState): CsourcesOpt =
+  let file = state.nimDir/"config/build_config.txt" # for newer nim versions, this file specifies correct csources_v1 to use
   if file.fileExists:
     let tab = file.readFile.parseKeyVal
-    result = opt.coptv1
+    result = state.coptv1
     result.rev = tab["nim_csourcesHash"]
-  elif gitIsAncestorOf(opt.nimDir, "a9b62de", opt.rev): # commit that introduced csources_v1
-    result = opt.coptv1
+  elif gitIsAncestorOf(state.nimDir, "a9b62de", state.rev): # commit that introduced csources_v1
+    result = state.coptv1
     result.rev = csourcesV1Revs[0]
   else:
-    let tag = gitLatestTag(opt.nimDir)
-    result = opt.coptv0
+    let tag = gitLatestTag(state.nimDir)
+    result = state.coptv0
     result.rev = tag.toCsourcesRev
-  result.nimCsourcesExe = toNimCsourcesExe(opt.binDir, result.name, result.rev)
+  result.nimCsourcesExe = toNimCsourcesExe(state.binDir, result.name, result.rev)
 
 proc main2(opt: DiggerOpt) =
   const
     csourcesName = "csources"
     csourcesV1Name = "csources_v1"
-  var opt = opt
+  let state = DiggerState(nimDir: opt.nimDir, rev: opt.rev)
   let nimdiggerHome = getEnv(NimDiggerEnv, getHomeDir() / ".nimdigger")
-  if opt.nimDir.len == 0:
-    opt.nimDir = nimdiggerHome / "cache/Nim"
-  if verbose: dbg opt
-  let nimDir = opt.nimDir
-  opt.binDir = nimDir/"bin"
-  let nimDiggerExe = opt.binDir / "nim_nimdigger"
+  if state.nimDir.len == 0:
+    state.nimDir = nimdiggerHome / "cache/Nim"
+  if verbose: dbg state
+  let nimDir = state.nimDir
+  state.binDir = nimDir/"bin"
+  let nimDiggerExe = state.binDir / "nim_nimdigger"
 
   if nimDir.dirExists:
     doAssert fileExists(nimDir / "lib/system.nim"), fmt"nimDir is not a nim repo: {nimDir}"
@@ -189,20 +188,20 @@ proc main2(opt: DiggerOpt) =
     createDir nimDir.parentDir
     gitClone("https://github.com/nim-lang/Nim", nimDir)
 
-  opt.coptv0 = CsourcesOpt(dir: nimDir/csourcesName, url: "https://github.com/nim-lang/csources.git", name: csourcesName, revs: csourcesRevs)
-  opt.coptv1 = CsourcesOpt(dir: nimDir/csourcesV1Name, url: "https://github.com/nim-lang/csources_v1.git", name: csourcesV1Name, revs: csourcesV1Revs)
-  for copt in [opt.coptv0, opt.coptv1]:
-    copt.binDir = opt.binDir
+  state.coptv0 = CsourcesOpt(dir: nimDir/csourcesName, url: "https://github.com/nim-lang/csources.git", name: csourcesName, revs: csourcesRevs)
+  state.coptv1 = CsourcesOpt(dir: nimDir/csourcesV1Name, url: "https://github.com/nim-lang/csources_v1.git", name: csourcesV1Name, revs: csourcesV1Revs)
+  for copt in [state.coptv0, state.coptv1]:
+    copt.binDir = state.binDir
     copt.fetch = opt.fetch
     if opt.buildAllCsources:
       buildCsourcesAnyRevs(copt)
 
   if opt.fetch: gitFetch(nimDir)
-  if opt.rev.len > 0: gitResetHard(nimDir, opt.rev)
-  else: opt.rev = "HEAD"
+  if state.rev.len > 0: gitResetHard(nimDir, state.rev)
+  else: state.rev = "HEAD"
 
   if opt.compileNim:
-    let copt = getNimCsourcesAnyExe(opt)
+    let copt = getNimCsourcesAnyExe(state)
     buildCsourcesRev(copt)
     # TODO: we could also cache those, optionally maybe (could get large?) or use this as hint in nim bisect to prefer those
     discard runCmdOutput(fmt"{copt.nimCsourcesExe} c -o:{nimDiggerExe} --hints:off --skipUserCfg compiler/nim.nim", nimDir)
@@ -210,13 +209,13 @@ proc main2(opt: DiggerOpt) =
   if opt.bisectCmd.len > 0:
     doAssert opt.oldrev.len > 0 # for regressions, aka goodrev
     doAssert opt.newrev.len > 0 # for a regressions, aka badrev 
-    runCmd(fmt"git -C {opt.nimDir.quoteShell} bisect start {opt.newrev} {opt.oldrev}")
+    runCmd(fmt"git -C {state.nimDir.quoteShell} bisect start {opt.newrev} {opt.oldrev}")
     let exe = getAppFileName()
     var msg2 = opt.bisectCmd
     if opt.bisectBugfix:
       msg2 = fmt"! ({msg2})" # negate exit code
     let bisectCmd2 = fmt"{exe} --compileNim && cp {nimDiggerExe.quoteShell} bin/nim && {msg2}" # TODO: inside () in case it does weird things?
-    runCmd(fmt"git -C {opt.nimDir.quoteShell} bisect run bash -c {bisectCmd2.quoteShell}")
+    runCmd(fmt"git -C {state.nimDir.quoteShell} bisect run bash -c {bisectCmd2.quoteShell}")
 
 proc main(rev = "", nimDir = "", compileNim = false, fetch = false, bisectCmd = "", newrev = "", oldrev = "", bisectBugfix = false, verbose = false) =
   nimdigger.verbose = verbose

From 3a28fd35ab274c5efedf7234a5d2c111e2f8db3b Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 28 May 2021 12:32:40 -0700
Subject: [PATCH 05/23] improve usage

---
 tools/nimdigger.nim | 57 ++++++++++++++++++++++++++++-----------------
 1 file changed, 35 insertions(+), 22 deletions(-)

diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
index e3abda9d9efb1..d45befd934e4a 100644
--- a/tools/nimdigger.nim
+++ b/tools/nimdigger.nim
@@ -2,6 +2,10 @@
 
 ## notes
 can build as far back as: v0.12.0~157
+
+##
+nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 -- bin/nim c --hints:off --skipparentcfg --skipusercfg $timn_D/tests/nim/all/t12329.nim
+
 ]#
 
 import std/[os, osproc, strformat, macros, strutils, tables, algorithm]
@@ -21,7 +25,8 @@ type
     # TODO: specify whether we should compile nim
     bisectCmd: string
     bisectBugfix: bool
-    oldrev, newrev: string
+    oldnew: string
+    args: seq[string]
   CsourcesOpt = ref object
     url: string
     dir: string
@@ -33,8 +38,7 @@ type
     name: string
     nimCsourcesExe: string
   DiggerState = ref object ## nimdigger internal state
-    coptv0: CsourcesOpt
-    coptv1: CsourcesOpt
+    coptv0, coptv1: CsourcesOpt
     binDir: string
     nimDir: string
     rev: string
@@ -170,12 +174,9 @@ proc getNimCsourcesAnyExe(state: DiggerState): CsourcesOpt =
   result.nimCsourcesExe = toNimCsourcesExe(state.binDir, result.name, result.rev)
 
 proc main2(opt: DiggerOpt) =
-  const
-    csourcesName = "csources"
-    csourcesV1Name = "csources_v1"
   let state = DiggerState(nimDir: opt.nimDir, rev: opt.rev)
-  let nimdiggerHome = getEnv(NimDiggerEnv, getHomeDir() / ".nimdigger")
   if state.nimDir.len == 0:
+    let nimdiggerHome = getEnv(NimDiggerEnv, getHomeDir() / ".nimdigger")
     state.nimDir = nimdiggerHome / "cache/Nim"
   if verbose: dbg state
   let nimDir = state.nimDir
@@ -187,14 +188,17 @@ proc main2(opt: DiggerOpt) =
   else:
     createDir nimDir.parentDir
     gitClone("https://github.com/nim-lang/Nim", nimDir)
-
-  state.coptv0 = CsourcesOpt(dir: nimDir/csourcesName, url: "https://github.com/nim-lang/csources.git", name: csourcesName, revs: csourcesRevs)
-  state.coptv1 = CsourcesOpt(dir: nimDir/csourcesV1Name, url: "https://github.com/nim-lang/csources_v1.git", name: csourcesV1Name, revs: csourcesV1Revs)
-  for copt in [state.coptv0, state.coptv1]:
-    copt.binDir = state.binDir
-    copt.fetch = opt.fetch
-    if opt.buildAllCsources:
-      buildCsourcesAnyRevs(copt)
+  block:
+    const
+      csourcesName = "csources"
+      csourcesV1Name = "csources_v1"
+    state.coptv0 = CsourcesOpt(dir: nimDir/csourcesName, url: "https://github.com/nim-lang/csources.git", name: csourcesName, revs: csourcesRevs)
+    state.coptv1 = CsourcesOpt(dir: nimDir/csourcesV1Name, url: "https://github.com/nim-lang/csources_v1.git", name: csourcesV1Name, revs: csourcesV1Revs)
+    for copt in [state.coptv0, state.coptv1]:
+      copt.binDir = state.binDir
+      copt.fetch = opt.fetch
+      if opt.buildAllCsources:
+        buildCsourcesAnyRevs(copt)
 
   if opt.fetch: gitFetch(nimDir)
   if state.rev.len > 0: gitResetHard(nimDir, state.rev)
@@ -206,20 +210,29 @@ proc main2(opt: DiggerOpt) =
     # TODO: we could also cache those, optionally maybe (could get large?) or use this as hint in nim bisect to prefer those
     discard runCmdOutput(fmt"{copt.nimCsourcesExe} c -o:{nimDiggerExe} --hints:off --skipUserCfg compiler/nim.nim", nimDir)
 
-  if opt.bisectCmd.len > 0:
-    doAssert opt.oldrev.len > 0 # for regressions, aka goodrev
-    doAssert opt.newrev.len > 0 # for a regressions, aka badrev 
-    runCmd(fmt"git -C {state.nimDir.quoteShell} bisect start {opt.newrev} {opt.oldrev}")
+  if opt.oldnew.len > 0:
+    let oldnew2 = opt.oldnew.split("..")
+    doAssert oldnew2.len == 2, opt.oldnew
+    let oldrev = oldnew2[0]
+    let newrev = oldnew2[1]
+    doAssert oldrev.len > 0 # for regressions, aka goodrev
+    doAssert newrev.len > 0 # for a regressions, aka badrev 
+    runCmd(fmt"git -C {state.nimDir.quoteShell} bisect start {newrev} {oldrev}")
     let exe = getAppFileName()
-    var msg2 = opt.bisectCmd
+    var msg2: string
+    if opt.bisectCmd.len > 0:
+      msg2 = opt.bisectCmd
+      doAssert opt.args.len == 0
+    else:
+      msg2 = opt.args.quoteShellCommand
     if opt.bisectBugfix:
       msg2 = fmt"! ({msg2})" # negate exit code
     let bisectCmd2 = fmt"{exe} --compileNim && cp {nimDiggerExe.quoteShell} bin/nim && {msg2}" # TODO: inside () in case it does weird things?
     runCmd(fmt"git -C {state.nimDir.quoteShell} bisect run bash -c {bisectCmd2.quoteShell}")
 
-proc main(rev = "", nimDir = "", compileNim = false, fetch = false, bisectCmd = "", newrev = "", oldrev = "", bisectBugfix = false, verbose = false) =
+proc main(rev = "", nimDir = "", compileNim = false, fetch = false, bisectCmd = "", oldnew = "", bisectBugfix = false, verbose = false, args: seq[string]) =
   nimdigger.verbose = verbose
-  main2(DiggerOpt.ctor(rev, nimDir, compileNim, fetch, bisectCmd, newrev, oldrev, bisectBugfix))
+  main2(DiggerOpt.ctor(rev, nimDir, compileNim, fetch, bisectCmd, oldnew, bisectBugfix, args))
 
 when isMainModule:
   import pkg/cligen

From a5a73b692fcfec73f574059cc4833a8f73ab3494 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 28 May 2021 12:43:53 -0700
Subject: [PATCH 06/23] _

---
 tools/nimdigger.nim | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
index d45befd934e4a..34e9a22f7cd78 100644
--- a/tools/nimdigger.nim
+++ b/tools/nimdigger.nim
@@ -5,7 +5,6 @@ can build as far back as: v0.12.0~157
 
 ##
 nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 -- bin/nim c --hints:off --skipparentcfg --skipusercfg $timn_D/tests/nim/all/t12329.nim
-
 ]#
 
 import std/[os, osproc, strformat, macros, strutils, tables, algorithm]
@@ -25,8 +24,8 @@ type
     # TODO: specify whether we should compile nim
     bisectCmd: string
     bisectBugfix: bool
-    oldnew: string
-    args: seq[string]
+    oldnew: string # eg: v0.20.0~10..v0.20.0
+    args: seq[string] # eg: bin/nim c --hints:off --skipparentcfg --skipusercfg $timn_D/tests/nim/all/t12329.nim 'arg1 bar' 'arg2'
   CsourcesOpt = ref object
     url: string
     dir: string
@@ -181,7 +180,6 @@ proc main2(opt: DiggerOpt) =
   if verbose: dbg state
   let nimDir = state.nimDir
   state.binDir = nimDir/"bin"
-  let nimDiggerExe = state.binDir / "nim_nimdigger"
 
   if nimDir.dirExists:
     doAssert fileExists(nimDir / "lib/system.nim"), fmt"nimDir is not a nim repo: {nimDir}"
@@ -204,6 +202,7 @@ proc main2(opt: DiggerOpt) =
   if state.rev.len > 0: gitResetHard(nimDir, state.rev)
   else: state.rev = "HEAD"
 
+  let nimDiggerExe = state.binDir / "nim_nimdigger"
   if opt.compileNim:
     let copt = getNimCsourcesAnyExe(state)
     buildCsourcesRev(copt)

From 238720d549605b28d5f63c567dca584b373df023 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 28 May 2021 15:06:05 -0700
Subject: [PATCH 07/23] add gitCleanDanger to avoid `git bisect` aborting when
 there are trashable untracked files

---
 tools/nimdigger.nim | 134 ++++++++++++++++++++++++++++----------------
 1 file changed, 86 insertions(+), 48 deletions(-)

diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
index 34e9a22f7cd78..a8dbaa9d27eb0 100644
--- a/tools/nimdigger.nim
+++ b/tools/nimdigger.nim
@@ -3,8 +3,22 @@
 ## notes
 can build as far back as: v0.12.0~157
 
-##
+## examples
+nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 --bisectCmd:'bin/nim -v | grep 0.19.0'
+66c0f7c3fb214485ca6cfd799af6e50798fcdf6d is the first REGRESSION commit
+
+nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 --bisectBugfix --bisectCmd:'bin/nim -v | grep 0.20.0'
+be9c38d2659496f918fb39e129b9b5b055eafd88 is the first BUGFIX commit
+
 nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 -- bin/nim c --hints:off --skipparentcfg --skipusercfg $timn_D/tests/nim/all/t12329.nim
+
+## note:
+https://stackoverflow.com/a/22592593/1426932 Magic exit statuses
+anything above 127 makes the bisection fail with something like:
+125 is magic and makes the run be skipped with git bisect skip.
+
+## TODO
+allow a way to verify that oldnew revisions honor what's implied by bisectBugfix:true|false
 ]#
 
 import std/[os, osproc, strformat, macros, strutils, tables, algorithm]
@@ -22,10 +36,9 @@ type
 
     # bisect cmds
     # TODO: specify whether we should compile nim
-    bisectCmd: string
-    bisectBugfix: bool
     oldnew: string # eg: v0.20.0~10..v0.20.0
-    args: seq[string] # eg: bin/nim c --hints:off --skipparentcfg --skipusercfg $timn_D/tests/nim/all/t12329.nim 'arg1 bar' 'arg2'
+    bisectCmd: string # eg: bin/nim c --hints:off --skipparentcfg --skipusercfg $timn_D/tests/nim/all/t12329.nim 'arg1 bar' 'arg2'
+    bisectBugfix: bool
   CsourcesOpt = ref object
     url: string
     dir: string
@@ -43,9 +56,10 @@ type
     rev: string
 
 const
-  csourcesRevs = "v0.9.4 v0.13.0 v0.15.2 v0.16.0 v0.17.0 v0.17.2 v0.18.0 v0.19.0 v0.20.0 64e3477".split
+  csourcesRevs = "v0.9.4 v0.13.0 v0.15.2 v0.16.0 v0.17.0 v0.17.2 v0.18.0 v0.19.0 v0.20.0 64e34778fa7e114b4afc753c7845dee250584167".split
   csourcesV1Revs = "a8a5241f9475099c823cfe1a5e0ca4022ac201ff".split
   NimDiggerEnv = "NIMDIGGER_HOME"
+  ExeExt2 = when ExeExt.len > 0: "." & ExeExt else: ""
 
 var verbose = false
 
@@ -77,10 +91,43 @@ macro ctor(obj: untyped, a: varargs[untyped]): untyped =
   result = nnkObjConstr.newTree(obj)
   for ai in a: result.add nnkExprColonExpr.newTree(ai, ai)
 
+proc parseKeyVal(a: string): OrderedTable[string, string] =
+  ## parse bash-like entries of the form key=val
+  let a2 = a.splitLines
+  for i, ai in a2:
+    if ai.len == 0 or ai.startsWith "#": continue
+    let b = split(ai, "=", maxsplit = 1)
+    doAssert b.len == 2, $(ai, b)
+    result[b[0]] = b[1]
+
 proc gitClone(url: string, dir: string) = runCmd fmt"git clone -q {url} {dir.quoteShell}"
 proc gitResetHard(dir: string, rev: string) = runCmd fmt"git -C {dir.quoteShell} reset --hard {rev}"
+proc gitCleanDanger(dir: string, requireConfirmation = true) =
+  #[
+  This is needed to avoid `git bisect` aborting with this error: The following untracked working tree files would be overwritten by checkout.
+  For example, this would happen in cases like this:
+  ```
+  cd $HOME/.nimdigger/cache/Nim
+  git checkout abaa42fd8a239ea62ddb39f6f58c3180137d750c
+  touch testament/testamenthtml.templ
+  cd -
+  nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 --bisectCmd:'bin/nim -v | grep 0.19.0'
+  ```
+  so we handle cleaning untracked files via dry run (-n) followed by -f if user confirms.
+  ]#
+  let files = runCmdOutput fmt"git -C {dir.quoteShell} clean -n"
+  if files.len > 0:
+    var runClean = true
+    if requireConfirmation:
+      echo &"untracked files may prevent `git bisect` from working, `git -C {dir.quoteShell} clean -n` returned:\n{files}"
+      echo fmt"enter `yes` to proceed with `git clean -f` in: {dir.quoteShell}"
+      let answer = stdin.readLine()
+      runClean = answer == "yes"
+    if runClean:
+      runCmd fmt"git -C {dir.quoteShell} clean -f"
 proc gitFetch(dir: string) = runCmd fmt"git -C {dir.quoteShell} fetch"
 proc gitLatestTag(dir: string): string = runCmdOutput("git describe --abbrev=0 HEAD", dir)
+proc gitCurrentRev(dir: string): string = runCmdOutput("git rev-parse HEAD", dir)
 proc gitCheck(dir: string) =
   # checks whether we're in a valid git repo; there may be better ways
   discard runCmdOutput("git describe HEAD", dir)
@@ -107,7 +154,7 @@ proc parseNimGitTag(tag: string): (int, int, int) =
 
 proc toNimCsourcesExe(binDir: string, name: string, rev: string): string =
   let rev2 = rev.replace(".", "_")
-  result = binDir / fmt"nim_nimdigger_{name}_{rev2}"
+  result = binDir / fmt"nim_nimdigger_{name}_{rev2}{ExeExt2}"
 
 proc buildCsourcesRev(copt: CsourcesOpt) =
   # sync with `_nimBuildCsourcesIfNeeded`
@@ -122,7 +169,7 @@ proc buildCsourcesRev(copt: CsourcesOpt) =
     let make = "gmake"
   else:
     let make = "make"
-  let oldNim = copt.binDir / "nim"
+  let oldNim = copt.binDir / "nim" & ExeExt2
   removeFile(oldNim) # otherwise `make` may incorrectly decide there's notthing to build
   let ncpu = countProcessors()
   if copt.rev.isGitNimTag and copt.rev.parseNimGitTag < (0,15,2):
@@ -133,29 +180,19 @@ proc buildCsourcesRev(copt: CsourcesOpt) =
   if isSimulate():
     dbg csourcesExe
   else:
-    copyFile(oldNim, csourcesExe) # TODO: windows: do i need to add exe or it's smart enough?
+    copyFile(oldNim, csourcesExe)
 
 proc buildCsourcesAnyRevs(copt: CsourcesOpt) =
   for rev in copt.revs:
     copt.rev = rev
     buildCsourcesRev(copt)
 
-proc parseKeyVal(a: string): OrderedTable[string, string] =
-  ## parse bash-like entries of the form key=val
-  let a2 = a.splitLines
-  for i, ai in a2:
-    if ai.len == 0 or ai.startsWith "#": continue
-    let b = split(ai, "=", maxsplit = 1)
-    doAssert b.len == 2, $(ai, b)
-    result[b[0]] = b[1]
-
 proc toCsourcesRev(rev: string): string =
   let ver = rev.parseNimGitTag
   if ver >= (1, 0, 0): return csourcesRevs[^1]
   for a in csourcesRevs[1 ..< ^1].reversed:
     if ver >= a.parseNimGitTag: return a
-  # v0.9.4 seems broken
-  return csourcesRevs[1]
+  return csourcesRevs[1] # because v0.9.4 seems broken
 
 proc getNimCsourcesAnyExe(state: DiggerState): CsourcesOpt =
   let file = state.nimDir/"config/build_config.txt" # for newer nim versions, this file specifies correct csources_v1 to use
@@ -186,28 +223,25 @@ proc main2(opt: DiggerOpt) =
   else:
     createDir nimDir.parentDir
     gitClone("https://github.com/nim-lang/Nim", nimDir)
-  block:
-    const
-      csourcesName = "csources"
-      csourcesV1Name = "csources_v1"
-    state.coptv0 = CsourcesOpt(dir: nimDir/csourcesName, url: "https://github.com/nim-lang/csources.git", name: csourcesName, revs: csourcesRevs)
-    state.coptv1 = CsourcesOpt(dir: nimDir/csourcesV1Name, url: "https://github.com/nim-lang/csources_v1.git", name: csourcesV1Name, revs: csourcesV1Revs)
-    for copt in [state.coptv0, state.coptv1]:
-      copt.binDir = state.binDir
-      copt.fetch = opt.fetch
-      if opt.buildAllCsources:
-        buildCsourcesAnyRevs(copt)
+  state.coptv0 = CsourcesOpt(dir: nimDir/"csources", url: "https://github.com/nim-lang/csources.git", name: "csources", revs: csourcesRevs)
+  state.coptv1 = CsourcesOpt(dir: nimDir/"csources_v1", url: "https://github.com/nim-lang/csources_v1.git", name: "csources_v1", revs: csourcesV1Revs)
+  for copt in [state.coptv0, state.coptv1]:
+    copt.binDir = state.binDir
+    copt.fetch = opt.fetch
+    if opt.buildAllCsources:
+      buildCsourcesAnyRevs(copt)
 
   if opt.fetch: gitFetch(nimDir)
-  if state.rev.len > 0: gitResetHard(nimDir, state.rev)
-  else: state.rev = "HEAD"
-
-  let nimDiggerExe = state.binDir / "nim_nimdigger"
+  if state.rev.len > 0:
+    gitResetHard(nimDir, state.rev)
+  state.rev = gitCurrentRev(state.nimDir)
+  let nimDiggerExe = state.binDir / fmt"nim_nimdigger_nim_{state.rev}{ExeExt2}"
   if opt.compileNim:
-    let copt = getNimCsourcesAnyExe(state)
-    buildCsourcesRev(copt)
-    # TODO: we could also cache those, optionally maybe (could get large?) or use this as hint in nim bisect to prefer those
-    discard runCmdOutput(fmt"{copt.nimCsourcesExe} c -o:{nimDiggerExe} --hints:off --skipUserCfg compiler/nim.nim", nimDir)
+    if not nimDiggerExe.fileExists:
+      let copt = getNimCsourcesAnyExe(state)
+      buildCsourcesRev(copt)
+      discard runCmdOutput(fmt"{copt.nimCsourcesExe} c -o:{nimDiggerExe} --hints:off --skipUserCfg compiler/nim.nim", nimDir)
+    copyFile(nimDiggerExe, state.binDir / "nim" & ExeExt2)
 
   if opt.oldnew.len > 0:
     let oldnew2 = opt.oldnew.split("..")
@@ -216,22 +250,26 @@ proc main2(opt: DiggerOpt) =
     let newrev = oldnew2[1]
     doAssert oldrev.len > 0 # for regressions, aka goodrev
     doAssert newrev.len > 0 # for a regressions, aka badrev 
-    runCmd(fmt"git -C {state.nimDir.quoteShell} bisect start {newrev} {oldrev}")
+    gitCleanDanger(state.nimDir, requireConfirmation = true)
+    proc bisectStart(old, new: string)=
+      runCmd(fmt"git -C {state.nimDir.quoteShell} bisect start --term-old {old} --term-new {new} {newrev} {oldrev}")
+    if opt.bisectBugfix: bisectStart("BROKEN", "BUGFIX")
+    else: bisectStart("WORKS", "REGRESSION")
     let exe = getAppFileName()
-    var msg2: string
-    if opt.bisectCmd.len > 0:
-      msg2 = opt.bisectCmd
-      doAssert opt.args.len == 0
-    else:
-      msg2 = opt.args.quoteShellCommand
+    var msg2 = opt.bisectCmd
     if opt.bisectBugfix:
       msg2 = fmt"! ({msg2})" # negate exit code
-    let bisectCmd2 = fmt"{exe} --compileNim && cp {nimDiggerExe.quoteShell} bin/nim && {msg2}" # TODO: inside () in case it does weird things?
+    let bisectCmd2 = fmt"{exe} --compileNim && {msg2}" # TODO: inside () in case it does weird things?
     runCmd(fmt"git -C {state.nimDir.quoteShell} bisect run bash -c {bisectCmd2.quoteShell}")
 
-proc main(rev = "", nimDir = "", compileNim = false, fetch = false, bisectCmd = "", oldnew = "", bisectBugfix = false, verbose = false, args: seq[string]) =
+proc main(rev = "", nimDir = "", compileNim = false, fetch = false, oldnew = "", bisectBugfix = false, verbose = false, bisectCmd = "", args: seq[string]) =
   nimdigger.verbose = verbose
-  main2(DiggerOpt.ctor(rev, nimDir, compileNim, fetch, bisectCmd, oldnew, bisectBugfix, args))
+  var bisectCmd = bisectCmd
+  if bisectCmd.len == 0:
+    bisectCmd = args.quoteShellCommand
+  else:
+    doAssert args.len == 0
+  main2(DiggerOpt.ctor(rev, nimDir, compileNim, fetch, bisectCmd, oldnew, bisectBugfix))
 
 when isMainModule:
   import pkg/cligen

From 87c96dec165a48f8786f4060f58ccc9cc0cb0615 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 28 May 2021 15:42:09 -0700
Subject: [PATCH 08/23] improve doc

---
 tools/nimdigger.nim | 59 ++++++++++++++++++++++++++++++++++++---------
 1 file changed, 48 insertions(+), 11 deletions(-)

diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
index a8dbaa9d27eb0..e2224a0fdbfe9 100644
--- a/tools/nimdigger.nim
+++ b/tools/nimdigger.nim
@@ -1,24 +1,60 @@
-#[
+##[
+`nimdigger` is a tool to build nim at any revision (including custom branches), taking
+care of details such as figuring out automatically the correct csources/csources_v1 revision to use.
 
-## notes
-can build as far back as: v0.12.0~157
+## design goals
+* ease of use: 1 liner for running `git bisect` workflows, or to build nim at past revisions
+* performance: via caching both csources built binaries, and intermediate nim binaries
+* lazyness: build artifacts on demand
+* go as far back as possible, currently oldest buildable nim version is v0.12.0~157
 
 ## examples
-nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 --bisectCmd:'bin/nim -v | grep 0.19.0'
+build at any revision >= v0.12.0~157
+```
+$ nim r tools/nimdigger.nim --compileNim --rev:v0.15.2~10
+$ $HOME/.nimdigger/cache/Nim/bin/nim -v
+Nim Compiler Version 0.15.2 (2021-05-28) [MacOSX: amd64] [...]
+```
+
+find a which commit introduced a regression
+```
+$ nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 --bisectCmd:'bin/nim -v | grep 0.19.0'
 66c0f7c3fb214485ca6cfd799af6e50798fcdf6d is the first REGRESSION commit
+```
 
-nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 --bisectBugfix --bisectCmd:'bin/nim -v | grep 0.20.0'
+find a which commit introduced a bugfix
+```
+$ nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 --bisectBugfix --bisectCmd:'bin/nim -v | grep 0.20.0'
 be9c38d2659496f918fb39e129b9b5b055eafd88 is the first BUGFIX commit
+```
 
-nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 -- bin/nim c --hints:off --skipparentcfg --skipusercfg $timn_D/tests/nim/all/t12329.nim
+find an actual regression, taken from https://github.com/nim-lang/Nim/issues/16376; copy this snippet to /tmp/t16376.nim:
+```nim
+type Matrix[T] = object
+  data: T
+proc randMatrix*[T](m, n: int, max: T): Matrix[T] = discard
+proc randMatrix*[T](m, n: int, x: Slice[T]): Matrix[T] = discard
+template randMatrix*[T](m, n: int): Matrix[T] = randMatrix[T](m, n, T(1.0))
+let B = randMatrix[float32](20, 10)
+```
+```
+$ nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 -- bin/nim c --hints:off --skipparentcfg --skipusercfg /tmp/t16376.nim
+fd16875561634e3ef24072631cf85eeead6213f2 is the first REGRESSION commit
+```
 
-## note:
-https://stackoverflow.com/a/22592593/1426932 Magic exit statuses
-anything above 127 makes the bisection fail with something like:
-125 is magic and makes the run be skipped with git bisect skip.
+## notes
+Unstable API, subject to change
+]##
 
+#[
 ## TODO
 allow a way to verify that oldnew revisions honor what's implied by bisectBugfix:true|false
+
+## note
+we should give exit code = 125 to commits where nim won't build, to skip over, see also:
+https://stackoverflow.com/a/22592593/1426932 (Magic exit statuses)
+> anything above 127 makes the bisection fail with something like:
+> 125 is magic and makes the run be skipped with git bisect skip.
 ]#
 
 import std/[os, osproc, strformat, macros, strutils, tables, algorithm]
@@ -83,7 +119,7 @@ proc runCmdOutput(cmd: string, dir = ""): string =
 
 macro ctor(obj: untyped, a: varargs[untyped]): untyped =
   ## Generates an object constructor call from a list of fields.
-  # xxx expose in some `fusion/macros` or std/macros; FACTOR with pr_fusion_globs PR
+  # xxx expose in std/sugar, factor with https://github.com/nim-lang/fusion/pull/32
   runnableExamples:
     type Foo = object
       a, b: int
@@ -100,6 +136,7 @@ proc parseKeyVal(a: string): OrderedTable[string, string] =
     doAssert b.len == 2, $(ai, b)
     result[b[0]] = b[1]
 
+# xxx move some of these to std/private/gitutils.nim
 proc gitClone(url: string, dir: string) = runCmd fmt"git clone -q {url} {dir.quoteShell}"
 proc gitResetHard(dir: string, rev: string) = runCmd fmt"git -C {dir.quoteShell} reset --hard {rev}"
 proc gitCleanDanger(dir: string, requireConfirmation = true) =

From ad7d95ef92b7b62221906d76d784405e7ca74905 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 28 May 2021 15:49:45 -0700
Subject: [PATCH 09/23] remove timn/dgs

---
 tools/nimdigger.nim | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
index e2224a0fdbfe9..67aa3ce6b8f99 100644
--- a/tools/nimdigger.nim
+++ b/tools/nimdigger.nim
@@ -58,7 +58,13 @@ https://stackoverflow.com/a/22592593/1426932 (Magic exit statuses)
 ]#
 
 import std/[os, osproc, strformat, macros, strutils, tables, algorithm]
-import timn/dbgs
+
+proc `$`(a: ref): string =
+  if a == nil: "nil" else: $a[]
+
+template dbg(args: varargs[untyped]): untyped =
+  # so users can swap in their own better logging until stdlib has one
+  echo args
 
 type
   DiggerOpt = object ## nimdigger input
@@ -300,7 +306,8 @@ proc main2(opt: DiggerOpt) =
     runCmd(fmt"git -C {state.nimDir.quoteShell} bisect run bash -c {bisectCmd2.quoteShell}")
 
 proc main(rev = "", nimDir = "", compileNim = false, fetch = false, oldnew = "", bisectBugfix = false, verbose = false, bisectCmd = "", args: seq[string]) =
-  nimdigger.verbose = verbose
+  # nimdigger.verbose = verbose
+  nimdigger.verbose = true
   var bisectCmd = bisectCmd
   if bisectCmd.len == 0:
     bisectCmd = args.quoteShellCommand

From 63ff4a4f5390df4b8e4f0e3009beaca8472371dc Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 28 May 2021 16:34:10 -0700
Subject: [PATCH 10/23] fixup

---
 tools/nimdigger.nim | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
index 67aa3ce6b8f99..a90d3ff8a66d9 100644
--- a/tools/nimdigger.nim
+++ b/tools/nimdigger.nim
@@ -27,8 +27,10 @@ find a which commit introduced a bugfix
 $ nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 --bisectBugfix --bisectCmd:'bin/nim -v | grep 0.20.0'
 be9c38d2659496f918fb39e129b9b5b055eafd88 is the first BUGFIX commit
 ```
+Note that this is fast (e.g. 3s) if intermediate nim binaries have already been built/cached in prior runs.
 
-find an actual regression, taken from https://github.com/nim-lang/Nim/issues/16376; copy this snippet to /tmp/t16376.nim:
+find an actual regression, e.g. for https://github.com/nim-lang/Nim/issues/16376,
+copy this snippet to /tmp/t16376.nim:
 ```nim
 type Matrix[T] = object
   data: T
@@ -77,7 +79,7 @@ type
     verbose: bool
 
     # bisect cmds
-    # TODO: specify whether we should compile nim
+    # TODO: allow user to not compile nim, for cases where it's not needed
     oldnew: string # eg: v0.20.0~10..v0.20.0
     bisectCmd: string # eg: bin/nim c --hints:off --skipparentcfg --skipusercfg $timn_D/tests/nim/all/t12329.nim 'arg1 bar' 'arg2'
     bisectBugfix: bool
@@ -109,7 +111,7 @@ proc isSimulate(): bool =
   defined(nimDiggerSimulate)
 
 proc runCmd(cmd: string) =
-  # TODO: allow `dir` param
+  # TODO: allow `dir` param (or use `runCmdOutput`)
   if isSimulate():
     dbg cmd
   else:
@@ -302,12 +304,11 @@ proc main2(opt: DiggerOpt) =
     var msg2 = opt.bisectCmd
     if opt.bisectBugfix:
       msg2 = fmt"! ({msg2})" # negate exit code
-    let bisectCmd2 = fmt"{exe} --compileNim && {msg2}" # TODO: inside () in case it does weird things?
+    let bisectCmd2 = fmt"{exe} --compileNim && ( {msg2} )"
     runCmd(fmt"git -C {state.nimDir.quoteShell} bisect run bash -c {bisectCmd2.quoteShell}")
 
 proc main(rev = "", nimDir = "", compileNim = false, fetch = false, oldnew = "", bisectBugfix = false, verbose = false, bisectCmd = "", args: seq[string]) =
-  # nimdigger.verbose = verbose
-  nimdigger.verbose = true
+  nimdigger.verbose = verbose
   var bisectCmd = bisectCmd
   if bisectCmd.len == 0:
     bisectCmd = args.quoteShellCommand

From 602f3076fa5e55174278fd91306d568cb7d28114 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 28 May 2021 18:08:04 -0700
Subject: [PATCH 11/23] build binaries with -d:release

---
 tools/nimdigger.nim | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
index a90d3ff8a66d9..dd148d7aeb5db 100644
--- a/tools/nimdigger.nim
+++ b/tools/nimdigger.nim
@@ -282,10 +282,12 @@ proc main2(opt: DiggerOpt) =
   state.rev = gitCurrentRev(state.nimDir)
   let nimDiggerExe = state.binDir / fmt"nim_nimdigger_nim_{state.rev}{ExeExt2}"
   if opt.compileNim:
-    if not nimDiggerExe.fileExists:
+    let isCached = nimDiggerExe.fileExists
+    echo fmt"digger getting nim: {nimDiggerExe} cached: {isCached}"
+    if not isCached:
       let copt = getNimCsourcesAnyExe(state)
       buildCsourcesRev(copt)
-      discard runCmdOutput(fmt"{copt.nimCsourcesExe} c -o:{nimDiggerExe} --hints:off --skipUserCfg compiler/nim.nim", nimDir)
+      discard runCmdOutput(fmt"{copt.nimCsourcesExe} c -o:{nimDiggerExe} -d:release --hints:off --skipUserCfg compiler/nim.nim", nimDir)
     copyFile(nimDiggerExe, state.binDir / "nim" & ExeExt2)
 
   if opt.oldnew.len > 0:

From 53ff747e536a87860577a79dde69c7250e7ddad4 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 28 May 2021 19:05:06 -0700
Subject: [PATCH 12/23] fix: nim doc tools/nimdigger.nim

---
 tools/nimdigger.nim | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
index dd148d7aeb5db..18026b510b514 100644
--- a/tools/nimdigger.nim
+++ b/tools/nimdigger.nim
@@ -10,27 +10,29 @@ care of details such as figuring out automatically the correct csources/csources
 
 ## examples
 build at any revision >= v0.12.0~157
-```
+```bash
 $ nim r tools/nimdigger.nim --compileNim --rev:v0.15.2~10
 $ $HOME/.nimdigger/cache/Nim/bin/nim -v
 Nim Compiler Version 0.15.2 (2021-05-28) [MacOSX: amd64] [...]
 ```
 
 find a which commit introduced a regression
-```
-$ nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 --bisectCmd:'bin/nim -v | grep 0.19.0'
+```bash
+$ nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 \
+  --bisectCmd:'bin/nim -v | grep 0.19.0'
 66c0f7c3fb214485ca6cfd799af6e50798fcdf6d is the first REGRESSION commit
 ```
 
 find a which commit introduced a bugfix
-```
-$ nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 --bisectBugfix --bisectCmd:'bin/nim -v | grep 0.20.0'
+```bash
+$ nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 --bisectBugfix \
+  --bisectCmd:'bin/nim -v | grep 0.20.0'
 be9c38d2659496f918fb39e129b9b5b055eafd88 is the first BUGFIX commit
 ```
 Note that this is fast (e.g. 3s) if intermediate nim binaries have already been built/cached in prior runs.
 
 find an actual regression, e.g. for https://github.com/nim-lang/Nim/issues/16376,
-copy this snippet to /tmp/t16376.nim:
+copy this snippet to /tmp/t16376.nim
 ```nim
 type Matrix[T] = object
   data: T
@@ -39,8 +41,9 @@ proc randMatrix*[T](m, n: int, x: Slice[T]): Matrix[T] = discard
 template randMatrix*[T](m, n: int): Matrix[T] = randMatrix[T](m, n, T(1.0))
 let B = randMatrix[float32](20, 10)
 ```
-```
-$ nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 -- bin/nim c --hints:off --skipparentcfg --skipusercfg /tmp/t16376.nim
+```bash
+$ nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 -- \
+  bin/nim c --hints:off --skipparentcfg --skipusercfg /tmp/t16376.nim
 fd16875561634e3ef24072631cf85eeead6213f2 is the first REGRESSION commit
 ```
 

From 3c8fbbf77d03f7884b53429215a53989553aa1ff Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 28 May 2021 19:26:05 -0700
Subject: [PATCH 13/23] tools now have their docs alongside stdlib, compiler

---
 config/nimdoc.cfg     |  3 +++
 tools/ci_generate.nim |  2 +-
 tools/index.nim       | 15 +++++++++++++++
 tools/kochdocs.nim    |  8 +++++---
 4 files changed, 24 insertions(+), 4 deletions(-)
 create mode 100644 tools/index.nim

diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg
index ed1b346a22f1c..fb7dfe0e5312b 100644
--- a/config/nimdoc.cfg
+++ b/config/nimdoc.cfg
@@ -139,6 +139,9 @@ doc.body_toc_group = """
       <li>
         <a href="compiler/$theindexhref">Compiler docs</a>
       </li>
+      <li>
+        <a href="tools/$theindexhref">Tools docs</a>
+      </li>
       <li>
         <a href="https://nim-lang.github.io/fusion/theindex.html">Fusion docs</a>
       </li>
diff --git a/tools/ci_generate.nim b/tools/ci_generate.nim
index 52b84f0d89553..add2041f7c7c2 100644
--- a/tools/ci_generate.nim
+++ b/tools/ci_generate.nim
@@ -5,7 +5,7 @@ duplication that could be removed.
 
 ## usage
 edit this file as needed and then re-generate via:
-```
+```bash
 nim r tools/ci_generate.nim
 ```
 ]##
diff --git a/tools/index.nim b/tools/index.nim
new file mode 100644
index 0000000000000..cadea2bead1bf
--- /dev/null
+++ b/tools/index.nim
@@ -0,0 +1,15 @@
+##[
+This module only exists to generate docs for `tools/`.
+
+## links
+* [main docs](../lib.html)
+* [compiler user guide](../nimc.html)
+* [Internals of the Nim Compiler](../intern.html)
+]##
+
+#[
+* see also `compiler/index.nim`
+* move src/fusion/docutils.nim to std/private/docutils so it can be reused here too
+]#
+
+import nimdigger, ci_generate, nimgrep
diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim
index f25564fad4d73..d362b105ddb52 100644
--- a/tools/kochdocs.nim
+++ b/tools/kochdocs.nim
@@ -243,6 +243,8 @@ proc buildDocPackages(nimArgs, destPath: string) =
   # xxx keep in sync with what's in $nim_prs_D/config/nimdoc.cfg, or, rather,
   # start using nims instead of nimdoc.cfg
   docProject(destPath/"compiler", extra, "compiler/index.nim")
+  docProject(destPath/"tools", extra & " --threads", "tools/index.nim")
+    # --threads needed for nimgrep
 
 proc buildDoc(nimArgs, destPath: string) =
   # call nim for the documentation:
@@ -318,9 +320,9 @@ proc buildDocsDir*(args: string, dir: string) =
   let args = nimArgs & " " & args
   let docHackJsSource = buildJS()
   createDir(dir)
-  buildDocSamples(args, dir)
-  buildDoc(args, dir) # bottleneck
-  copyFile(dir / "overview.html", dir / "index.html")
+  # buildDocSamples(args, dir)
+  # buildDoc(args, dir) # bottleneck
+  # copyFile(dir / "overview.html", dir / "index.html")
   buildDocPackages(args, dir)
   copyFile(docHackJsSource, dir / docHackJsSource.lastPathPart)
 

From c7f1732603da88afbfeaedb6275d60c74963ae18 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 28 May 2021 19:30:14 -0700
Subject: [PATCH 14/23] improve doc wording

---
 compiler/index.nim  | 2 +-
 tools/index.nim     | 2 +-
 tools/nimdigger.nim | 3 ++-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/compiler/index.nim b/compiler/index.nim
index 2c2a34fb57bbd..696d1e6cafe59 100644
--- a/compiler/index.nim
+++ b/compiler/index.nim
@@ -1,5 +1,5 @@
 ##[
-This module only exists to generate docs for the compiler.
+This module only exists to generate internal docs for the compiler.
 
 ## links
 * [main docs](../lib.html)
diff --git a/tools/index.nim b/tools/index.nim
index cadea2bead1bf..a7ac065c8934c 100644
--- a/tools/index.nim
+++ b/tools/index.nim
@@ -1,5 +1,5 @@
 ##[
-This module only exists to generate docs for `tools/`.
+This module only exists to generate internal docs for `tools/`.
 
 ## links
 * [main docs](../lib.html)
diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
index 18026b510b514..5c4adf618b2f7 100644
--- a/tools/nimdigger.nim
+++ b/tools/nimdigger.nim
@@ -48,7 +48,8 @@ fd16875561634e3ef24072631cf85eeead6213f2 is the first REGRESSION commit
 ```
 
 ## notes
-Unstable API, subject to change
+* this uses `git` (in particular `bisect`), `csources`, `csources_v`, `bash`, `make`/`gmake`
+* Unstable API, subject to change
 ]##
 
 #[

From b1585f5ba07fc7effebbaf7eb83e412d0078d9b6 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Fri, 28 May 2021 20:36:36 -0700
Subject: [PATCH 15/23] fixup

---
 tools/kochdocs.nim | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim
index d362b105ddb52..bd5e0780deb5b 100644
--- a/tools/kochdocs.nim
+++ b/tools/kochdocs.nim
@@ -320,9 +320,9 @@ proc buildDocsDir*(args: string, dir: string) =
   let args = nimArgs & " " & args
   let docHackJsSource = buildJS()
   createDir(dir)
-  # buildDocSamples(args, dir)
-  # buildDoc(args, dir) # bottleneck
-  # copyFile(dir / "overview.html", dir / "index.html")
+  buildDocSamples(args, dir)
+  buildDoc(args, dir) # bottleneck
+  copyFile(dir / "overview.html", dir / "index.html")
   buildDocPackages(args, dir)
   copyFile(docHackJsSource, dir / docHackJsSource.lastPathPart)
 

From f80b6ecd24d704b3e1d8f905e67fffe7a9d35c74 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Sat, 29 May 2021 01:06:06 -0700
Subject: [PATCH 16/23] msg2=>msg

---
 tools/nimdigger.nim | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
index 5c4adf618b2f7..74300e19e6a81 100644
--- a/tools/nimdigger.nim
+++ b/tools/nimdigger.nim
@@ -307,10 +307,10 @@ proc main2(opt: DiggerOpt) =
     if opt.bisectBugfix: bisectStart("BROKEN", "BUGFIX")
     else: bisectStart("WORKS", "REGRESSION")
     let exe = getAppFileName()
-    var msg2 = opt.bisectCmd
+    var msg = opt.bisectCmd
     if opt.bisectBugfix:
-      msg2 = fmt"! ({msg2})" # negate exit code
-    let bisectCmd2 = fmt"{exe} --compileNim && ( {msg2} )"
+      msg = fmt"! ({msg})" # negate exit code
+    let bisectCmd2 = fmt"{exe} --compileNim && ( {msg} )"
     runCmd(fmt"git -C {state.nimDir.quoteShell} bisect run bash -c {bisectCmd2.quoteShell}")
 
 proc main(rev = "", nimDir = "", compileNim = false, fetch = false, oldnew = "", bisectBugfix = false, verbose = false, bisectCmd = "", args: seq[string]) =

From 550aec6ddfd7681ec9cb0837046d11f71f976385 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Sat, 29 May 2021 01:20:37 -0700
Subject: [PATCH 17/23] improve docs/naming a bit

---
 tools/nimdigger.nim | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
index 74300e19e6a81..9b1ba5134280f 100644
--- a/tools/nimdigger.nim
+++ b/tools/nimdigger.nim
@@ -87,21 +87,21 @@ type
     oldnew: string # eg: v0.20.0~10..v0.20.0
     bisectCmd: string # eg: bin/nim c --hints:off --skipparentcfg --skipusercfg $timn_D/tests/nim/all/t12329.nim 'arg1 bar' 'arg2'
     bisectBugfix: bool
-  CsourcesOpt = ref object
+  CsourcesState = ref object ## represents csources or csources_v1 repos
     url: string
-    dir: string
+    dir: string # e.g. /pathto/Nim/csources
     rev: string
     binDir: string
-    csourcesBuildArgs: string
+    csourcesBuildArgs: string ## extra args to build csources
     revs: seq[string]
     fetch: bool
     name: string
     nimCsourcesExe: string
   DiggerState = ref object ## nimdigger internal state
-    coptv0, coptv1: CsourcesOpt
-    binDir: string
-    nimDir: string
-    rev: string
+    nimDir: string # e.g.: /pathto/Nim
+    binDir: string # e.g.: $nimDir/bin
+    rev: string # e.g.: hash obtained from `git rev-parse HEAD`
+    csourceV0, csourceV1: CsourcesState
 
 const
   csourcesRevs = "v0.9.4 v0.13.0 v0.15.2 v0.16.0 v0.17.0 v0.17.2 v0.18.0 v0.19.0 v0.20.0 64e34778fa7e114b4afc753c7845dee250584167".split
@@ -205,7 +205,7 @@ proc toNimCsourcesExe(binDir: string, name: string, rev: string): string =
   let rev2 = rev.replace(".", "_")
   result = binDir / fmt"nim_nimdigger_{name}_{rev2}{ExeExt2}"
 
-proc buildCsourcesRev(copt: CsourcesOpt) =
+proc buildCsourcesRev(copt: CsourcesState) =
   # sync with `_nimBuildCsourcesIfNeeded`
   let csourcesExe = toNimCsourcesExe(copt.binDir, copt.name, copt.rev)
   if csourcesExe.fileExists:
@@ -231,7 +231,7 @@ proc buildCsourcesRev(copt: CsourcesOpt) =
   else:
     copyFile(oldNim, csourcesExe)
 
-proc buildCsourcesAnyRevs(copt: CsourcesOpt) =
+proc buildCsourcesAnyRevs(copt: CsourcesState) =
   for rev in copt.revs:
     copt.rev = rev
     buildCsourcesRev(copt)
@@ -243,18 +243,18 @@ proc toCsourcesRev(rev: string): string =
     if ver >= a.parseNimGitTag: return a
   return csourcesRevs[1] # because v0.9.4 seems broken
 
-proc getNimCsourcesAnyExe(state: DiggerState): CsourcesOpt =
+proc getCsourcesState(state: DiggerState): CsourcesState =
   let file = state.nimDir/"config/build_config.txt" # for newer nim versions, this file specifies correct csources_v1 to use
   if file.fileExists:
     let tab = file.readFile.parseKeyVal
-    result = state.coptv1
+    result = state.csourceV1
     result.rev = tab["nim_csourcesHash"]
   elif gitIsAncestorOf(state.nimDir, "a9b62de", state.rev): # commit that introduced csources_v1
-    result = state.coptv1
+    result = state.csourceV1
     result.rev = csourcesV1Revs[0]
   else:
     let tag = gitLatestTag(state.nimDir)
-    result = state.coptv0
+    result = state.csourceV0
     result.rev = tag.toCsourcesRev
   result.nimCsourcesExe = toNimCsourcesExe(state.binDir, result.name, result.rev)
 
@@ -272,9 +272,9 @@ proc main2(opt: DiggerOpt) =
   else:
     createDir nimDir.parentDir
     gitClone("https://github.com/nim-lang/Nim", nimDir)
-  state.coptv0 = CsourcesOpt(dir: nimDir/"csources", url: "https://github.com/nim-lang/csources.git", name: "csources", revs: csourcesRevs)
-  state.coptv1 = CsourcesOpt(dir: nimDir/"csources_v1", url: "https://github.com/nim-lang/csources_v1.git", name: "csources_v1", revs: csourcesV1Revs)
-  for copt in [state.coptv0, state.coptv1]:
+  state.csourceV0 = CsourcesState(dir: nimDir/"csources", url: "https://github.com/nim-lang/csources.git", name: "csources", revs: csourcesRevs)
+  state.csourceV1 = CsourcesState(dir: nimDir/"csources_v1", url: "https://github.com/nim-lang/csources_v1.git", name: "csources_v1", revs: csourcesV1Revs)
+  for copt in [state.csourceV0, state.csourceV1]:
     copt.binDir = state.binDir
     copt.fetch = opt.fetch
     if opt.buildAllCsources:
@@ -289,7 +289,7 @@ proc main2(opt: DiggerOpt) =
     let isCached = nimDiggerExe.fileExists
     echo fmt"digger getting nim: {nimDiggerExe} cached: {isCached}"
     if not isCached:
-      let copt = getNimCsourcesAnyExe(state)
+      let copt = getCsourcesState(state)
       buildCsourcesRev(copt)
       discard runCmdOutput(fmt"{copt.nimCsourcesExe} c -o:{nimDiggerExe} -d:release --hints:off --skipUserCfg compiler/nim.nim", nimDir)
     copyFile(nimDiggerExe, state.binDir / "nim" & ExeExt2)

From e3c1f6a69121ed12d0c94be931c4c6c276df057d Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Sat, 29 May 2021 01:22:21 -0700
Subject: [PATCH 18/23] address a comment regarding csourcesRevs

---
 tools/nimdigger.nim | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
index 9b1ba5134280f..a757e130ffa56 100644
--- a/tools/nimdigger.nim
+++ b/tools/nimdigger.nim
@@ -104,7 +104,8 @@ type
     csourceV0, csourceV1: CsourcesState
 
 const
-  csourcesRevs = "v0.9.4 v0.13.0 v0.15.2 v0.16.0 v0.17.0 v0.17.2 v0.18.0 v0.19.0 v0.20.0 64e34778fa7e114b4afc753c7845dee250584167".split
+  csourcesRevs = "v0.9.4 v0.13.0 v0.15.2 v0.16.0 v0.17.0 v0.17.2 v0.18.0 v0.19.0 v0.20.0".split &
+    "64e34778fa7e114b4afc753c7845dee250584167"
   csourcesV1Revs = "a8a5241f9475099c823cfe1a5e0ca4022ac201ff".split
   NimDiggerEnv = "NIMDIGGER_HOME"
   ExeExt2 = when ExeExt.len > 0: "." & ExeExt else: ""

From 7cc8d5cd7d2b822db0eaa87c2dd7e42839b42288 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Sat, 29 May 2021 01:24:02 -0700
Subject: [PATCH 19/23] fixup

---
 tools/nimdigger.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
index a757e130ffa56..024eb926d89b0 100644
--- a/tools/nimdigger.nim
+++ b/tools/nimdigger.nim
@@ -150,7 +150,7 @@ proc parseKeyVal(a: string): OrderedTable[string, string] =
     result[b[0]] = b[1]
 
 # xxx move some of these to std/private/gitutils.nim
-proc gitClone(url: string, dir: string) = runCmd fmt"git clone -q {url} {dir.quoteShell}"
+proc gitClone(url: string, dir: string) = runCmd fmt"git clone -q {url.quoteShell} {dir.quoteShell}"
 proc gitResetHard(dir: string, rev: string) = runCmd fmt"git -C {dir.quoteShell} reset --hard {rev}"
 proc gitCleanDanger(dir: string, requireConfirmation = true) =
   #[

From f806b91fd54a8e5d7d88c145293c57bc3823c9e3 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Sat, 29 May 2021 09:47:46 -0700
Subject: [PATCH 20/23] simplify

---
 tools/nimdigger.nim | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
index 024eb926d89b0..421490a7eb062 100644
--- a/tools/nimdigger.nim
+++ b/tools/nimdigger.nim
@@ -142,12 +142,11 @@ macro ctor(obj: untyped, a: varargs[untyped]): untyped =
 
 proc parseKeyVal(a: string): OrderedTable[string, string] =
   ## parse bash-like entries of the form key=val
-  let a2 = a.splitLines
-  for i, ai in a2:
+  for ai in a.splitLines:
     if ai.len == 0 or ai.startsWith "#": continue
-    let b = split(ai, "=", maxsplit = 1)
-    doAssert b.len == 2, $(ai, b)
-    result[b[0]] = b[1]
+    let kv = split(ai, "=", maxsplit = 1)
+    doAssert kv.len == 2, $(ai, kv)
+    result[kv[0]] = kv[1]
 
 # xxx move some of these to std/private/gitutils.nim
 proc gitClone(url: string, dir: string) = runCmd fmt"git clone -q {url.quoteShell} {dir.quoteShell}"

From 5debdaa29c7d54c1cfed44a213b877b7fd98f62d Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Thu, 3 Jun 2021 16:05:19 -0700
Subject: [PATCH 21/23] use strscans

---
 tests/tools/tnimdigger.nim | 11 +++++++++++
 tools/nimdigger.nim        | 23 ++++++++++-------------
 2 files changed, 21 insertions(+), 13 deletions(-)
 create mode 100644 tests/tools/tnimdigger.nim

diff --git a/tests/tools/tnimdigger.nim b/tests/tools/tnimdigger.nim
new file mode 100644
index 0000000000000..d2d307277b2d9
--- /dev/null
+++ b/tests/tools/tnimdigger.nim
@@ -0,0 +1,11 @@
+import tools/nimdigger {.all.}
+
+block: # parseNimGitTag
+  doAssert parseNimGitTag("v1.4.2") == (1, 4, 2)
+  doAssertRaises(ValueError): discard parseNimGitTag("v1.4")
+  doAssertRaises(ValueError): discard parseNimGitTag("v1.4.2a")
+  doAssertRaises(ValueError): discard parseNimGitTag("av1.4.2")
+
+block: # isGitNimTag
+  doAssert isGitNimTag("v1.4.2")
+  doAssert not isGitNimTag("v1.4.2a")
diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
index 421490a7eb062..401e07f0657d4 100644
--- a/tools/nimdigger.nim
+++ b/tools/nimdigger.nim
@@ -185,21 +185,18 @@ proc gitIsAncestorOf(dir: string, rev1, rev2: string): bool =
   gitCheck(dir)
   execShellCmd(fmt"git -C {dir.quoteShell} merge-base --is-ancestor {rev1} {rev2}") == 0
 
-proc isGitNimTag(tag: string): bool =
-  if not tag.startsWith "v":
-    return false
-  let ver = tag[1..^1].split(".")
-  return ver.len == 3
+import std/strscans
 
 proc parseNimGitTag(tag: string): (int, int, int) =
-  doAssert tag.isGitNimTag, tag
-  let ver = tag[1..^1].split(".")
-  template impl(i) =
-    # improve pending https://github.com/nim-lang/Nim/pull/18038
-    result[i] = ver[i].parseInt
-  impl 0
-  impl 1
-  impl 2
+  if not scanf(tag, "v$i.$i.$i$.", result[0], result[1], result[2]):
+    raise newException(ValueError, tag)
+
+proc isGitNimTag(tag: string): bool =
+  try:
+    discard parseNimGitTag(tag)
+    return true
+  except ValueError:
+    return false
 
 proc toNimCsourcesExe(binDir: string, name: string, rev: string): string =
   let rev2 = rev.replace(".", "_")

From 8cb3e43e1a6e9d6647cfb76437fb5e112d9b9170 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Thu, 3 Jun 2021 16:07:16 -0700
Subject: [PATCH 22/23] ctor => construct

---
 tools/nimdigger.nim | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
index 401e07f0657d4..ce55f84fd6ffb 100644
--- a/tools/nimdigger.nim
+++ b/tools/nimdigger.nim
@@ -130,13 +130,13 @@ proc runCmdOutput(cmd: string, dir = ""): string =
   result = outp
   stripLineEnd(result)
 
-macro ctor(obj: untyped, a: varargs[untyped]): untyped =
+macro construct(obj: untyped, a: varargs[untyped]): untyped =
   ## Generates an object constructor call from a list of fields.
   # xxx expose in std/sugar, factor with https://github.com/nim-lang/fusion/pull/32
   runnableExamples:
     type Foo = object
       a, b: int
-    doAssert Foo.ctor(a,b) == Foo(a: a, b: b)
+    doAssert Foo.construct(a,b) == Foo(a: a, b: b)
   result = nnkObjConstr.newTree(obj)
   for ai in a: result.add nnkExprColonExpr.newTree(ai, ai)
 
@@ -317,7 +317,7 @@ proc main(rev = "", nimDir = "", compileNim = false, fetch = false, oldnew = "",
     bisectCmd = args.quoteShellCommand
   else:
     doAssert args.len == 0
-  main2(DiggerOpt.ctor(rev, nimDir, compileNim, fetch, bisectCmd, oldnew, bisectBugfix))
+  main2(DiggerOpt.construct(rev, nimDir, compileNim, fetch, bisectCmd, oldnew, bisectBugfix))
 
 when isMainModule:
   import pkg/cligen

From bffca2eaad19573b25d492bf14d319a16791791e Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Thu, 3 Jun 2021 16:17:23 -0700
Subject: [PATCH 23/23] use getCacheDir("nimdigger") instead of
 ~/.nimdigger/cache

---
 tools/nimdigger.nim | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
index ce55f84fd6ffb..a7b1713ae2420 100644
--- a/tools/nimdigger.nim
+++ b/tools/nimdigger.nim
@@ -12,7 +12,7 @@ care of details such as figuring out automatically the correct csources/csources
 build at any revision >= v0.12.0~157
 ```bash
 $ nim r tools/nimdigger.nim --compileNim --rev:v0.15.2~10
-$ $HOME/.nimdigger/cache/Nim/bin/nim -v
+$ $NIMDIGGER_CACHE/Nim/bin/nim -v
 Nim Compiler Version 0.15.2 (2021-05-28) [MacOSX: amd64] [...]
 ```
 
@@ -107,7 +107,7 @@ const
   csourcesRevs = "v0.9.4 v0.13.0 v0.15.2 v0.16.0 v0.17.0 v0.17.2 v0.18.0 v0.19.0 v0.20.0".split &
     "64e34778fa7e114b4afc753c7845dee250584167"
   csourcesV1Revs = "a8a5241f9475099c823cfe1a5e0ca4022ac201ff".split
-  NimDiggerEnv = "NIMDIGGER_HOME"
+  NimDiggerEnv = "NIMDIGGER_CACHE"
   ExeExt2 = when ExeExt.len > 0: "." & ExeExt else: ""
 
 var verbose = false
@@ -156,7 +156,7 @@ proc gitCleanDanger(dir: string, requireConfirmation = true) =
   This is needed to avoid `git bisect` aborting with this error: The following untracked working tree files would be overwritten by checkout.
   For example, this would happen in cases like this:
   ```
-  cd $HOME/.nimdigger/cache/Nim
+  cd $NIMDIGGER_CACHE/Nim
   git checkout abaa42fd8a239ea62ddb39f6f58c3180137d750c
   touch testament/testamenthtml.templ
   cd -
@@ -258,8 +258,8 @@ proc getCsourcesState(state: DiggerState): CsourcesState =
 proc main2(opt: DiggerOpt) =
   let state = DiggerState(nimDir: opt.nimDir, rev: opt.rev)
   if state.nimDir.len == 0:
-    let nimdiggerHome = getEnv(NimDiggerEnv, getHomeDir() / ".nimdigger")
-    state.nimDir = nimdiggerHome / "cache/Nim"
+    let nimdiggerCache = getEnv(NimDiggerEnv, getCacheDir("nimdigger"))
+    state.nimDir = nimdiggerCache / "Nim"
   if verbose: dbg state
   let nimDir = state.nimDir
   state.binDir = nimDir/"bin"