fixes #1987; show template calls as inlined frames when debugging

src/hastur.nim

@@ -1838,6 +1838,91 @@ proc runNativeCodegenTests*(dir: string; overwrite: bool) =
   else:
     echo "SUCCESS."
 
+proc extractDebugMetadata(llText, testName: string): string =
+  ## Keep only the debug metadata lines of a `.ll` file, in order. Everything
+  ## else (instructions, types, mangled temporaries) churns with unrelated
+  ## codegen changes and would make the golden useless as a debug-info guard.
+  ##
+  ## Two things are normalized so the golden is portable across checkouts:
+  ## `directory:` holds an absolute path, and module-init/`main` symbols carry
+  ## the module-hash suffix, which depends on the absolute source path.
+  result = ""
+  for line in llText.splitLines:
+    var s = line.strip
+    if not s.startsWith("!"): continue
+    if not ("DILocation" in s or "DISubprogram" in s or
+            "DILocalVariable" in s or "DIFile" in s): continue
+    # `directory: "<abs path>"` -> `directory: "<dir>"`
+    let dirPos = s.find("directory: \"")
+    if dirPos >= 0:
+      let valStart = dirPos + "directory: \"".len
+      let valEnd = s.find('"', valStart)
+      if valEnd > valStart:
+        s = s[0 ..< valStart] & "<dir>" & s[valEnd .. ^1]
+    # `X60Qini_0_<modulehash>` -> `X60Qini_0_<mod>`
+    let sufPos = s.find("X60Qini_0_")
+    if sufPos >= 0:
+      let valStart = sufPos + "X60Qini_0_".len
+      var valEnd = valStart
+      while valEnd < s.len and s[valEnd] notin {'"'}: inc valEnd
+      s = s[0 ..< valStart] & "<mod>" & s[valEnd .. ^1]
+    result.add s
+    result.add "\n"
+
+proc runLLVMDebugTests*(dir: string; overwrite: bool) =
+  ## Golden suite over the LLVM backend's *debug metadata*. For each `.nim`,
+  ## compile with `nimony l` and diff the DWARF-relevant metadata lines of the
+  ## emitted `.ll` against a checked-in `<test>.ll.expected`.
+  ##
+  ## Only `DIFile`/`DISubprogram`/`DILocation`/`DILocalVariable` lines are
+  ## compared: those carry the inlined-frame structure (#1987) and are stable,
+  ## whereas the surrounding IR churns with every unrelated codegen change.
+  ##
+  ## `hastur.mode = skip` — the LLVM backend cannot build the full stdlib yet,
+  ## so these are opt-in via `hastur tests/llvmdebug`. Add `--overwrite` to
+  ## regenerate the goldens after an intended debug-info change.
+  if not skipBuild:
+    buildNimonyToolchain()
+    buildLengc()
+  let t0 = epochTime()
+  var c = TestCounters(total: 0, failures: 0)
+  var files: seq[string] = @[]
+  for x in walkDir(dir):
+    if x.kind == pcFile and x.path.endsWith(".nim") and
+       x.path.extractFilename != "setup.nim":
+      files.add x.path
+  sort files
+  for file in files:
+    inc c.total
+    let cacheArg =
+      if nimcacheDir != "nimcache": "--nimcache:" & quoteShell(nimcacheDir) & " "
+      else: ""
+    # The LLVM backend cannot link the full stdlib yet, so the exit code is not
+    # meaningful here; the `.ll` is written before linking and is what we check.
+    discard execLocal("nimony", "l --silentMake --isMain " & cacheArg &
+      quoteShell(file))
+    let llFile = generatedFile(file, ".ll")
+    if not llFile.fileExists():
+      failure c, file, "lengc .ll", "missing: " & llFile
+      continue
+    let actual = extractDebugMetadata(readFile(llFile), file.splitFile.name)
+    let expectedFile = file.changeFileExt(".ll.expected")
+    if overwrite:
+      writeFile(expectedFile, actual)
+    elif not expectedFile.fileExists():
+      failure c, file, "golden debug metadata", "missing: " & expectedFile
+    else:
+      let expected = readFile(expectedFile)
+      if expected.strip != actual.strip:
+        failure c, file, expected, actual
+  echo c.total - c.failures, " / ", c.total,
+    " llvm-debug tests successful in ",
+    formatFloat(epochTime() - t0, ffDecimal, precision=2), "s."
+  if c.failures > 0:
+    quit "FAILURE: Some llvm-debug tests failed."
+  else:
+    echo "SUCCESS."
+
 # ---- deterministic self-host bootstrap ------------------------------------
 # `bin0/` is a fresh copy of the host-Nim-built toolchain in `bin/`; `binN/`
 # (N >= 1) is `binN-1/`'s nimony recompiling all three self-tools from

src/lengc/codegen.nim

@@ -196,7 +196,7 @@ proc errorAt(m: MainModule; msg: string; n: Cursor) {.noreturn.} =
   ## (mangled symbol plus embedded line info), which says nothing a reader wants.
   let info = rawLineInfo(n)
   if info.isValid:
-    write stdout, m.pool.filenames[info.file]
+    write stdout, realFile(m.pool.filenames[info.file])
     write stdout, "(" & $info.line & ", " & $(info.col+1) & ") "
   # `Error: `, not the `[Error] ` of the rendering `error` above: this is a
   # user-facing diagnostic, and that is the spelling every other user-facing
@@ -210,7 +210,7 @@ proc errorAt(m: MainModule; msg: string; n: Cursor) {.noreturn.} =
 proc error(m: MainModule; msg: string; n: Cursor) {.noreturn.} =
   let info = rawLineInfo(n)
   if info.isValid:
-    write stdout, m.pool.filenames[info.file]
+    write stdout, realFile(m.pool.filenames[info.file])
     write stdout, "(" & $info.line & ", " & $(info.col+1) & ") "
   write stdout, "[Error] "
   write stdout, msg

src/lengc/llvmcodegen.nim

@@ -71,6 +71,9 @@ type
     diBasicTypeCache*: Table[string, int] # "i32"/"float"/… -> DIBasicType id
     compositeTypeDone*: HashSet[SymId]    # symIds with fully built DICompositeType
     globalExprs*: seq[int] # DIGlobalVariableExpression IDs for DICompileUnit globals
+    fileIdsByName*: Table[string, int]  # source path -> DIFile meta
+    inlineSpCache*: Table[string, int] # expanded routine (mangled) -> spId
+    nullSigId*: int # shared `!DISubroutineType(types: !{null})` for template SPs
 
 type
   PrimTypes* = object
@@ -124,7 +127,7 @@ proc errorAt(m: MainModule; msg: string; n: Cursor) {.noreturn.} =
   ## what is wrong in prose (the render would append the raw mangled symbol).
   let info = rawLineInfo(n)
   if info.isValid:
-    write stdout, m.pool.filenames[info.file]
+    write stdout, realFile(m.pool.filenames[info.file])
     write stdout, "(" & $info.line & ", " & $(info.col+1) & ") "
   # `Error: `, not the `[Error] ` of the rendering `error` above: this is a
   # user-facing diagnostic, and that is the spelling every other user-facing
@@ -138,7 +141,7 @@ proc errorAt(m: MainModule; msg: string; n: Cursor) {.noreturn.} =
 proc error(m: MainModule; msg: string; n: Cursor) {.noreturn.} =
   let info = rawLineInfo(n)
   if info.isValid:
-    write stdout, m.pool.filenames[info.file]
+    write stdout, realFile(m.pool.filenames[info.file])
     write stdout, "(" & $info.line & ", " & $(info.col+1) & ") "
   write stdout, "[Error] "
   write stdout, msg

src/lengc/llvmdebug.nim

@@ -56,43 +56,140 @@ proc genDIBasicType(c: var LLVMCode; name: string; sizeBits,
     ", encoding: " & $encoding & ")")
   c.debug.diBasicTypeCache[key] = result
 
-proc getOrCreateDIFile(c: var LLVMCode; fid: FileId): int =
-  ## Get or create a DIFile metadata node for the given FileId.
-  let key = int(fid)
-  # `getOrDefault` rather than `[]`: the latter is `.raises` (KeyError) under
-  # Nimony. Metadata ids are positive, so -1 is an unambiguous "not cached".
-  let cached = c.debug.fileIds.getOrDefault(key, -1)
+proc getOrCreateDIFileByName(c: var LLVMCode; path: string): int =
+  ## Get or create a DIFile metadata node for a plain source path. Keyed on the
+  ## path itself, since the declaration sites carried in a forged filename have
+  ## no `FileId` of their own.
+  let cached = c.debug.fileIdsByName.getOrDefault(path, -1)
   if cached >= 0:
     return cached
-  let path = c.m.pool.filenames[fid]
   let (dir, name, ext) = splitFile(path)
   let fullName = name & ext
   let directory = absoluteDirOrQuit(dir)
   result = c.addMetadata("!DIFile(filename: \"" & fullName &
       "\", directory: \"" & directory & "\")")
-  c.debug.fileIds[key] = result
+  c.debug.fileIdsByName[path] = result
   # First real source file → create the compile unit using this file
   if c.debug.cuId == 0:
     c.debug.cuId = c.addMetadata("distinct !DICompileUnit(language: DW_LANG_C99, file: !" &
       $result & ", producer: \"lengc\", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug)")
 
-proc dbgLocation(c: var LLVMCode; info: NifLineInfo): string =
-  ## Return a `, !dbg !N` suffix for the given source location, or "" if invalid.
-  if not info.isValid: return ""
+proc getOrCreateDIFile(c: var LLVMCode; fid: FileId): int =
+  ## Get or create a DIFile metadata node for the given FileId.
+  let key = int(fid)
+  # `getOrDefault` rather than `[]`: the latter is `.raises` (KeyError) under
+  # Nimony. Metadata ids are positive, so -1 is an unambiguous "not cached".
+  let cached = c.debug.fileIds.getOrDefault(key, -1)
+  if cached >= 0:
+    return cached
+  # A forged filename carries template-expansion provenance (`__crucial\0…`);
+  # the DIFile must name the real source, and `dbgLocationId` reads the chain
+  # separately to build the inlined frames.
+  result = getOrCreateDIFileByName(c, realFile(c.m.pool.filenames[fid]))
+  c.debug.fileIds[key] = result
+
+proc getOrCreateInlineSP(c: var LLVMCode; origin: CrucialOrigin;
+                         fallbackFileId, fallbackLine: int): int
+
+proc dbgLocationId(c: var LLVMCode; info: NifLineInfo): int =
+  ## Build a DILocation metadata node for the given source location and return
+  ## its id, or 0 if the location is invalid.
+  ##
+  ## When the location's filename carries expansion provenance
+  ## (`__crucial\0<outer>\0<inner>\0real.nim`, see `comesfrom`), the chain is
+  ## turned into DWARF inlined frames: one synthetic DISubprogram per expanded
+  ## routine, each location scoped to the innermost and chained outwards through
+  ## `inlinedAt` until it reaches the enclosing proc. That is what makes a
+  ## debugger show a template as an inlined frame instead of jumping into its
+  ## definition file (#1987).
+  if not info.isValid: return 0
   let rawInfo = info
-  if not rawInfo.file.isValid: return ""
+  if not rawInfo.file.isValid: return 0
   let fileId = getOrCreateDIFile(c, rawInfo.file)
-  let scopeId =
+
+  # The call site: the location this expansion was written at, in the enclosing
+  # proc. Everything the chain builds hangs off it.
+  proc scopeInProc(c: var LLVMCode; fileId: int): int =
     if fileId == c.currentProc.subprogramFileId:
       c.currentProc.subprogramId
     else:
       c.addMetadata("!DILexicalBlockFile(scope: !" &
           $c.currentProc.subprogramId &
         ", file: !" & $fileId & ", discriminator: 0)")
-  let locId = c.addMetadata("!DILocation(line: " & $rawInfo.line &
+
+  let fname = c.m.pool.filenames[rawInfo.file]
+  if not isCrucialFile(fname):
+    result = c.addMetadata("!DILocation(line: " & $rawInfo.line &
+      ", column: " & $(rawInfo.col + 1) &
+      ", scope: !" & $scopeInProc(c, fileId) & ")")
+    return
+
+  # Outermost first, so each iteration nests one level deeper. The call-site
+  # location is the same for every level: expansions carry no separate location
+  # for where each nested template was invoked, so the whole chain attributes
+  # back to the statement the outermost expansion replaced.
+  var inlinedAt = c.addMetadata("!DILocation(line: " & $rawInfo.line &
     ", column: " & $(rawInfo.col + 1) &
-    ", scope: !" & $scopeId & ")")
-  result = ", !dbg !" & $locId
+    ", scope: !" & $scopeInProc(c, fileId) & ")")
+  var scopeId = 0
+  for origin in crucialOrigins(fname):
+    let spId = getOrCreateInlineSP(c, origin, fileId, rawInfo.line)
+    if spId == 0: continue
+    if scopeId != 0:
+      # The previous level's location becomes this one's call site.
+      inlinedAt = c.addMetadata("!DILocation(line: " & $rawInfo.line &
+        ", column: " & $(rawInfo.col + 1) &
+        ", scope: !" & $scopeId &
+        ", inlinedAt: !" & $inlinedAt & ")")
+    scopeId = spId
+  if scopeId == 0:
+    result = inlinedAt
+  else:
+    result = c.addMetadata("!DILocation(line: " & $rawInfo.line &
+      ", column: " & $(rawInfo.col + 1) &
+      ", scope: !" & $scopeId &
+      ", inlinedAt: !" & $inlinedAt & ")")
+
+proc dbgLocation(c: var LLVMCode; info: NifLineInfo): string =
+  ## Return a `, !dbg !N` suffix for the given source location, or "" if invalid.
+  let locId = dbgLocationId(c, info)
+  result = if locId == 0: "" else: ", !dbg !" & $locId
+
+proc getOrCreateInlineSP(c: var LLVMCode; origin: CrucialOrigin;
+                         fallbackFileId, fallbackLine: int): int =
+  ## Get or create the synthetic DISubprogram for an expanded routine, shared by
+  ## every call site of it. Returns 0 when none can be built.
+  ##
+  ## The frame is placed at the routine's *declaration* site, which the forged
+  ## filename carries for exactly this reason: a template declaration does not
+  ## survive into Leng, and the expanded code's own line info points at whatever
+  ## file the body came from. The fallbacks cover a chain written before the
+  ## declaration site was encoded.
+  let cached = c.debug.inlineSpCache.getOrDefault(origin.sym, -1)
+  if cached >= 0: return cached
+  var fileId = fallbackFileId
+  var line = fallbackLine
+  if origin.declFile.len > 0:
+    fileId = getOrCreateDIFileByName(c, origin.declFile)
+    line = int(origin.declLine)
+  if fileId == 0: return 0
+  # A subprogram definition needs a type and a unit; the signature is opaque
+  # (`!{null}`: unknown return, no formals) because a template has neither a
+  # calling convention nor materialized parameters.
+  if c.debug.nullSigId == 0:
+    c.debug.nullSigId = c.addMetadata("!DISubroutineType(types: !{null})")
+  var isGlobal = false
+  var shortName = extractBasename(origin.sym, isGlobal)
+  if shortName.len == 0: shortName = origin.sym
+  result = c.addMetadata("distinct !DISubprogram(name: \"" &
+    shortName &
+    "\", scope: !" & $fileId &
+    ", file: !" & $fileId &
+    ", line: " & $line &
+    ", type: !" & $c.debug.nullSigId &
+    ", scopeLine: " & $line &
+    ", spFlags: DISPFlagDefinition, unit: !" & $c.debug.cuId & ")")
+  c.debug.inlineSpCache[origin.sym] = result
 
 proc createSubprogram(c: var LLVMCode; name: string; info: NifLineInfo): int =
   ## Create a DISubprogram metadata node for a function.
@@ -147,18 +244,35 @@ proc emitDbgDeclare(c: var LLVMCode; localName: string; symId: SymId;
     useType = genDIBasicType(c, "int " & $bits, bits, DW_ATE_signed)
   let debugName = if wasName.len > 0: wasName else: nifSymBaseName(c, symId)
   let fileId = getOrCreateDIFile(c, rawInfo.file)
+  # A variable declared inside a template expansion belongs to that template's
+  # synthetic subprogram, and its declare location must carry the matching
+  # `inlinedAt` chain: LLVM's verifier rejects a #dbg_declare whose variable
+  # scope and location scope disagree. `template t = (var x = ...)` is ordinary
+  # code, so this is required, not polish.
+  #
+  # Both come from the same place - the forged filename - so they cannot drift:
+  # the innermost origin names the SP that `dbgLocationId` will scope to.
+  let fname = c.m.pool.filenames[rawInfo.file]
+  let inFrame = isCrucialFile(fname)
+  var varScopeId = c.currentProc.subprogramId
+  if inFrame:
+    for origin in crucialOrigins(fname):
+      let spId = getOrCreateInlineSP(c, origin, fileId, rawInfo.line)
+      if spId != 0: varScopeId = spId
   var varMetadata = "!DILocalVariable(name: \"" & debugName & "\""
   if argNo > 0:
     varMetadata.add ", arg: " & $argNo
-  varMetadata.add ", scope: !" & $c.currentProc.subprogramId &
+  varMetadata.add ", scope: !" & $varScopeId &
     ", file: !" & $fileId &
     ", line: " & $rawInfo.line &
     ", type: !" & $useType & ")"
   let varId = c.addMetadata(varMetadata)
-  c.currentProc.retainedNodes.add varId
-  let locId = c.addMetadata("!DILocation(line: " & $rawInfo.line &
-    ", column: " & $(rawInfo.col + 1) &
-    ", scope: !" & $c.currentProc.subprogramId & ")")
+  if not inFrame:
+    # `retainedNodes` lists the variables scoped to *this* subprogram; one
+    # scoped to a template's synthetic SP does not belong in the proc's list.
+    c.currentProc.retainedNodes.add varId
+  let locId = dbgLocationId(c, info)
+  if locId == 0: return
   c.emitRaw "#dbg_declare(ptr " & localName & ", !" & $varId &
       ", !DIExpression(), !" & $locId & ")"