name and enforce the operand-headed statement shape

doc/tags.md

@@ -140,7 +140,7 @@
 | `(was STR)` | LengPragma | |
 | `(selectany)` | LengPragma, NimonyPragma | |
 | `(pragmas (pragma ...)*)` | LengOther, NimonyOther, NimonyStmt, NiflerKind | begin of pragma section |
-| `(pragmax X (pragmas ...))` | NimonyExpr, NimonyStmt, NiflerKind | pragma expressions |
+| `(pragmax ^(pragmas ...) X)` | NimonyExpr, NimonyStmt, NiflerKind | pragma expressions. Transparent: introduces no scope and no semantics, so a consumer that does not care descends into the body. The `^` marks the pragma list as an operand, not a statement: a statement walker MUST step over it before recursing, or it walks the pragmas as code. See `nimony_model.OperandHeadedS` / `bodyInto` |
 | `(align X)` | LengPragma, NimonyPragma | |
 | `(bits X)`| LengPragma, NimonyPragma | |
 | `(vector)` | LengPragma | |
@@ -350,6 +350,41 @@
 | `(assembler)` | NimonyPragma, LengPragma | the `{.assembler.}` **proc pragma** (no children): every construct in the body maps one-to-one to assembler, in source order, with no temporaries invented and no operand materialised. The back end (arkham) owns that checking — see `nativenif/doc/intrinsics.md` §8. Spelled `assembler` rather than `asm` because Nim's parser reads a pragma entry as an expression and so cannot accept a keyword there; it is unrelated to the `(asm X+)` statement |
 | `(deferexpansion)` | NimonyOther | emitted by a template *plugin* as its entire output to say "I cannot answer while the argument still contains type variables — ask me again after instantiation". The compiler then parks the sem-checked call in the tree as `(at <template> <args>…)` instead of replacing it with an expansion — `(at …)` because that is the only unresolved type application a type slot accepts. `subsGenericProc` substitutes into it like any other type, and the instantiation's re-sem turns it back into a call, which drives the plugin again with concrete types. Rejected (a hard error) when no argument contains a generic parameter, which is what makes the retry well-founded |
 | `(needtypes SYM+)` | NimonyOther | emitted by a template *plugin* as its entire output to ask the compiler for the declarations of the named symbols. The compiler appends them to the plugin's second input (`loadTypeDefinitions()`) and runs the plugin again. This is how a plugin resolves a nominal type: it arrives in the main input as an opaque `Symbol`, and a plugin runs in its own process with no way to look it up. Only what is asked for is shipped, so a plugin that never asks pays nothing. Requesting a symbol that was already provided is a hard error, which is what bounds the loop |
+| `(comesfrom ^SYM S*)` | LengStmt, NimonyStmt | a statement list that came from expanding `SYM` — today a template, and the same shape suits any inliner. Carried so the debug backend can emit it as an inlined frame (DWARF `DISubprogram` + `inlinedAt`). Transparent like `(par ...)` is for expressions: it introduces **no scope** and no semantics, so a consumer that does not care descends into the body and skips the rest. The `^` marks `SYM` as an operand, not a statement: a statement walker MUST step over it before recursing, or it walks the origin symbol as code. See `nimony_model.OperandHeadedS` / `bodyInto`. Only statement-position (void) expansions are wrapped |
+
+### Child slot notation
+
+A slot in the first column names the kind of child expected there: `D` a
+symbol definition, `Y` a symbol use, `T` a type, `X` an expression, `S` a
+statement, `P` a pragma list, `LIT`/`STR`/`INTLIT` a literal. A nested
+`(tag ...)` means that literal tag. `...` means the form is underspecified.
+
+Modifiers attach to a slot:
+
+| modifier | meaning |
+|---|---|
+| `.X` | a `DotToken` is also allowed in this slot |
+| `X?` | the slot is optional |
+| `X*`, `X+` | the slot repeats |
+| `^X` | **transparent operand** — see below |
+
+`^` marks the leading child of a *transparent* wrapper: a tag that introduces
+no scope and no semantics, so a pass that does not care about it descends
+straight into the body — but whose first child is an operand rather than a
+statement. A walker that recurses into every child as a statement walks that
+operand and corrupts the tree, silently, with the damage surfacing several
+passes later.
+
+Two tags carry it, `(pragmax ^(pragmas ...) X)` and `(comesfrom ^SYM S*)`.
+The marker exists because the slot *kind* cannot express this: `(bind Y)` has
+the same shape as `(comesfrom ^SYM S*)`, and `(when ...)` the same shape as
+`(pragmax ^(pragmas ...) X)`, with opposite walking contracts. `(block .D X)`
+also leads with an operand and is deliberately unmarked — it opens a scope and
+is a `break` target, so it is not transparent.
+
+`src/validator` reads the marker and reports a `case n.stmtKind` branch that
+opens such a tag and walks every child. In the compiler, use
+`nimony_model.bodyInto` (or `bodyStart`), which steps over the slot for you.
 
 ### unpackflat, unpacktup, unpackdecl
 

src/hastur.nim

@@ -1621,6 +1621,91 @@ proc runNativeCodegenTests*(dir: string; overwrite: bool) =
   else:
     echo "SUCCESS."
 
+proc extractDebugMetadata(llText, testName: string): string =
+  ## Keep only the debug metadata lines of a `.ll` file, in order. Everything
+  ## else (instructions, types, mangled temporaries) churns with unrelated
+  ## codegen changes and would make the golden useless as a debug-info guard.
+  ##
+  ## Two things are normalized so the golden is portable across checkouts:
+  ## `directory:` holds an absolute path, and module-init/`main` symbols carry
+  ## the module-hash suffix, which depends on the absolute source path.
+  result = ""
+  for line in llText.splitLines:
+    var s = line.strip
+    if not s.startsWith("!"): continue
+    if not ("DILocation" in s or "DISubprogram" in s or
+            "DILocalVariable" in s or "DIFile" in s): continue
+    # `directory: "<abs path>"` -> `directory: "<dir>"`
+    let dirPos = s.find("directory: \"")
+    if dirPos >= 0:
+      let valStart = dirPos + "directory: \"".len
+      let valEnd = s.find('"', valStart)
+      if valEnd > valStart:
+        s = s[0 ..< valStart] & "<dir>" & s[valEnd .. ^1]
+    # `X60Qini_0_<modulehash>` -> `X60Qini_0_<mod>`
+    let sufPos = s.find("X60Qini_0_")
+    if sufPos >= 0:
+      let valStart = sufPos + "X60Qini_0_".len
+      var valEnd = valStart
+      while valEnd < s.len and s[valEnd] notin {'"'}: inc valEnd
+      s = s[0 ..< valStart] & "<mod>" & s[valEnd .. ^1]
+    result.add s
+    result.add "\n"
+
+proc runLLVMDebugTests*(dir: string; overwrite: bool) =
+  ## Golden suite over the LLVM backend's *debug metadata*. For each `.nim`,
+  ## compile with `nimony l` and diff the DWARF-relevant metadata lines of the
+  ## emitted `.ll` against a checked-in `<test>.ll.expected`.
+  ##
+  ## Only `DIFile`/`DISubprogram`/`DILocation`/`DILocalVariable` lines are
+  ## compared: those carry the inlined-frame structure (#1987) and are stable,
+  ## whereas the surrounding IR churns with every unrelated codegen change.
+  ##
+  ## `hastur.mode = skip` — the LLVM backend cannot build the full stdlib yet,
+  ## so these are opt-in via `hastur tests/llvmdebug`. Add `--overwrite` to
+  ## regenerate the goldens after an intended debug-info change.
+  if not skipBuild:
+    buildNimonyToolchain()
+    buildLengc()
+  let t0 = epochTime()
+  var c = TestCounters(total: 0, failures: 0)
+  var files: seq[string] = @[]
+  for x in walkDir(dir):
+    if x.kind == pcFile and x.path.endsWith(".nim") and
+       x.path.extractFilename != "setup.nim":
+      files.add x.path
+  sort files
+  for file in files:
+    inc c.total
+    let cacheArg =
+      if nimcacheDir != "nimcache": "--nimcache:" & quoteShell(nimcacheDir) & " "
+      else: ""
+    # The LLVM backend cannot link the full stdlib yet, so the exit code is not
+    # meaningful here; the `.ll` is written before linking and is what we check.
+    discard execLocal("nimony", "l --silentMake --isMain " & cacheArg &
+      quoteShell(file))
+    let llFile = generatedFile(file, ".ll")
+    if not llFile.fileExists():
+      failure c, file, "lengc .ll", "missing: " & llFile
+      continue
+    let actual = extractDebugMetadata(readFile(llFile), file.splitFile.name)
+    let expectedFile = file.changeFileExt(".ll.expected")
+    if overwrite:
+      writeFile(expectedFile, actual)
+    elif not expectedFile.fileExists():
+      failure c, file, "golden debug metadata", "missing: " & expectedFile
+    else:
+      let expected = readFile(expectedFile)
+      if expected.strip != actual.strip:
+        failure c, file, expected, actual
+  echo c.total - c.failures, " / ", c.total,
+    " llvm-debug tests successful in ",
+    formatFloat(epochTime() - t0, ffDecimal, precision=2), "s."
+  if c.failures > 0:
+    quit "FAILURE: Some llvm-debug tests failed."
+  else:
+    echo "SUCCESS."
+
 # ---- deterministic self-host bootstrap ------------------------------------
 # `bin0/` is a fresh copy of the host-Nim-built toolchain in `bin/`; `binN/`
 # (N >= 1) is `binN-1/`'s nimony recompiling all three self-tools from

src/hexer/constparams.nim

@@ -514,7 +514,7 @@ proc tr(c: var Context; dest: var TokenBuf; n: var Cursor) =
          ImportasS, FromimportS, ImportexceptS, ExportS, ExportexceptS,
          CommentS, DiscardS, UnpackdeclS, AssumeS, AssertS, CallstrlitS,
          InfixS, PrefixS, HcallS, StaticstmtS, BindS, MixinS, UsingS,
-         AsmS, DeferS, NoStmt:
+         AsmS, DeferS, ComesfromS, NoStmt:
         # generic container: copy the head and recurse into the children
         copyInto dest, n:
           while n.hasMore: tr c, dest, n

src/hexer/coro_transform.nim

@@ -1274,7 +1274,7 @@ proc trGoto*(c: var Context; dest: var TokenBuf; n: var Cursor) =
             DiscardS, TryS, RaiseS, UnpackdeclS, AssumeS,
             AssertS, CallstrlitS, InfixS, PrefixS, HcallS,
             StaticstmtS, BindS, MixinS, UsingS, AsmS,
-            DeferS, NoStmt:
+            DeferS, ComesfromS, NoStmt:
           dest.addParLe(n.cursorTagId, n.info)
           n.into:
             while n.hasMore:
@@ -1883,7 +1883,7 @@ proc coroTr*(c: var Context; dest: var TokenBuf; n: var Cursor) =
         ExportexceptS, DiscardS, TryS, UnpackdeclS,
         AssumeS, AssertS, CallstrlitS, InfixS, PrefixS,
         HcallS, StaticstmtS, BindS, MixinS, UsingS,
-        AsmS, DeferS, NoStmt:
+        AsmS, DeferS, ComesfromS, NoStmt:
       case n.exprKind
       of CallKinds - {DelayX}:
         trCall c, dest, n

src/hexer/destroyer.nim

@@ -513,7 +513,7 @@ proc tr(c: var Context; n: var Cursor) =
         FromimportS, ImportexceptS, ExportS, ExportexceptS,
         CommentS, DiscardS, UnpackdeclS, AssumeS, AssertS,
         CallstrlitS, InfixS, PrefixS, HcallS, StaticstmtS,
-        BindS, MixinS, UsingS, AsmS, DeferS, NoStmt:
+        BindS, MixinS, UsingS, AsmS, DeferS, ComesfromS, NoStmt:
       if n.isTagLit:
         let isStmtList = n.stmtKind == StmtsS
         c.dest.addParLe(n.cursorTagId, n.info)

src/hexer/desugar.nim

@@ -1119,7 +1119,7 @@ proc tr(c: var Context; dest: var TokenBuf; n: var Cursor; isTopScope = false) =
           YldS, PragmaxS, ImportasS, ExportexceptS, DiscardS,
           TryS, RaiseS, UnpackdeclS, AssumeS, AssertS,
           CallstrlitS, InfixS, PrefixS, HcallS, StaticstmtS,
-          BindS, MixinS, UsingS, AsmS, DeferS:
+          BindS, MixinS, UsingS, AsmS, DeferS, ComesfromS:
         trSons(c, dest, n)
     of SetconstrX:
       genSetConstr(c, dest, n)

src/hexer/duplifier.nim

@@ -695,7 +695,7 @@ proc trOnlyEssentials(c: var Context; n: var Cursor)
           CommentS, DiscardS, TryS, RaiseS, UnpackdeclS, AssumeS,
           AssertS, CallstrlitS, InfixS, PrefixS, HcallS,
           StaticstmtS, BindS, MixinS, UsingS, AsmS, DeferS,
-          NoStmt:
+          ComesfromS, NoStmt:
         # generic statement: copy the head and recurse into the children
         copyInto c.dest, n:
           while n.hasMore: trOnlyEssentials c, n

src/hexer/eraiser.nim

@@ -224,7 +224,7 @@ proc tr(c: var Context; dest: var TokenBuf; n: var Cursor) =
          FromimportS, ImportexceptS, ExportS, ExportexceptS, CommentS,
          DiscardS, TryS, RaiseS, UnpackdeclS, AssumeS, AssertS,
          CallstrlitS, InfixS, PrefixS, HcallS, StaticstmtS, BindS,
-         MixinS, UsingS, AsmS, DeferS, NoStmt:
+         MixinS, UsingS, AsmS, DeferS, ComesfromS, NoStmt:
         # generic container: copy the head and recurse into the children
         copyInto dest, n:
           while n.hasMore:

src/hexer/iterinliner.nim

@@ -295,7 +295,7 @@ proc inlineLoopBody(e: var EContext; dest: var TokenBuf; c: var Cursor; mapping:
         ExportexceptS, CommentS, DiscardS, TryS, RaiseS,
         UnpackdeclS, AssumeS, AssertS, CallstrlitS, InfixS,
         PrefixS, HcallS, StaticstmtS, BindS, MixinS, UsingS,
-        AsmS, DeferS, CoroforS, NoStmt:
+        AsmS, DeferS, CoroforS, ComesfromS, NoStmt:
       if c.substructureKind == KvU:
         # In KvU: first element is field name, don't substitute it
         takeInto dest, c:
@@ -359,7 +359,7 @@ proc inlineIteratorBody(e: var EContext; dest: var TokenBuf;
         ImportexceptS, ExportS, ExportexceptS, CommentS, DiscardS,
         TryS, RaiseS, UnpackdeclS, AssumeS, AssertS, CallstrlitS,
         InfixS, PrefixS, HcallS, StaticstmtS, BindS, MixinS, UsingS,
-        AsmS, DeferS, NoStmt:
+        AsmS, DeferS, ComesfromS, NoStmt:
       takeInto dest, c:
         while c.hasMore:
           inlineIteratorBody(e, dest, c, forStmt, yieldType)
@@ -398,7 +398,7 @@ proc replaceSymbol(e: var EContext; dest: var TokenBuf; c: var Cursor; relations
         ExportS, ExportexceptS, CommentS, DiscardS, TryS, RaiseS,
         UnpackdeclS, AssumeS, AssertS, CallstrlitS, InfixS,
         PrefixS, HcallS, StaticstmtS, BindS, MixinS, UsingS,
-        AsmS, DeferS, NoStmt:
+        AsmS, DeferS, ComesfromS, NoStmt:
       if c.substructureKind == KvU:
         # In KvU: first element is field name, don't substitute it
         takeInto dest, c:
@@ -923,7 +923,7 @@ proc transformStmt(e: var EContext; dest: var TokenBuf; c: var Cursor) =
         ImportexceptS, ExportS, ExportexceptS, CommentS, DiscardS,
         TryS, RaiseS, UnpackdeclS, AssumeS, AssertS, CallstrlitS,
         InfixS, PrefixS, HcallS, StaticstmtS, BindS, MixinS,
-        UsingS, AsmS, DeferS, CoroforS, NoStmt:
+        UsingS, AsmS, DeferS, CoroforS, ComesfromS, NoStmt:
       takeInto dest, c:
         while c.hasMore:
           transformStmt(e, dest, c)

src/hexer/lambdalifting.nim

@@ -325,7 +325,7 @@ proc tr(c: var Context; dest: var TokenBuf; n: var Cursor) =
       ExportexceptS, DiscardS, TryS, RaiseS, UnpackdeclS,
       AssumeS, AssertS, CallstrlitS, InfixS, PrefixS, HcallS,
       StaticstmtS, BindS, MixinS, UsingS, AsmS, DeferS,
-      NoStmt:
+      ComesfromS, NoStmt:
       case n.exprKind
       of CallKinds:
         trCall c, dest, n
@@ -1235,7 +1235,7 @@ proc tre(c: var Context; dest: var TokenBuf; n: var Cursor) =
       ExportexceptS, DiscardS, TryS, RaiseS, UnpackdeclS,
       AssumeS, AssertS, CallstrlitS, InfixS, PrefixS, HcallS,
       StaticstmtS, BindS, MixinS, UsingS, AsmS, DeferS,
-      NoStmt:
+      ComesfromS, NoStmt:
       case n.exprKind
       of CallKinds:
         genCall(c, dest, n)

src/hexer/lengcgen.nim

@@ -2047,6 +2047,14 @@ proc trStmt(c: var EContext; dest: var TokenBuf; n: var Cursor; mode = TraverseI
         takeInto dest, n:
           while n.hasMore:
             trStmt c, dest, n, mode
+    of ComesfromS:
+      # Survives into Leng so the LLVM debug backend can emit an inlined frame.
+      # No `openScope`: the wrapper is transparent, like `stmts` and unlike
+      # `scope`. The origin symbol is copied verbatim as the first child.
+      takeInto dest, n:
+        takeTree dest, n # the origin symbol
+        while n.hasMore:
+          trStmt c, dest, n, mode
     of ScopeS:
       c.typeCache.openScope()
       if mode == TraverseTopLevel:
@@ -2575,46 +2583,64 @@ proc initHasCall(c: var EContext; n: Cursor): bool =
   # Now at the init value; scan its subtree
   result = scanInitValue(c, n)
 
+proc trToplevelItem(c: var EContext; dest: var TokenBuf; n: var Cursor) =
+  let sk = n.stmtKind
+  if sk in {GvarS, GletS, TvarS, TletS}:
+    let tag = if sk in {TvarS, TletS}: TvarY else: GvarY
+    if not initHasCall(c, n):
+      # Simple init (literal, nil, etc.): keep at top level.
+      # NIFC can emit "Type var = value;" at C file scope directly.
+      trLocal c, dest, n, tag, TraverseAll, SymId(0)
+    else:
+      # Complex init with function calls: emit a no-init declaration at top
+      # level and place the actual init as an assignment inside the Init proc
+      # body so that any temp variables created by to_stmts remain in scope.
+      let savedN = n
+      trLocal c, dest, n, tag, TraverseSig, SymId(0)
+      var initN = savedN
+      inc initN  # past gvar/glet tag -> at SymbolDef
+      let (initSym, initInfo) = getSymDef(c, initN)
+      skipExportMarker c, initN
+      skip initN  # past pragmas -> at type
+      skip initN  # past type -> at init value
+      swap dest, c.initBody
+      dest.addParLe AsgnS, initInfo
+      dest.addSymUse(initSym, initInfo)
+      trExpr c, dest, initN
+      dest.addParRi()
+      swap dest, c.initBody
+  elif sk == StmtsS:
+    # Nested stmts block: recurse to handle mixed decls and executable code
+    n.into:
+      while n.hasMore:
+        trToplevelItem c, dest, n
+  elif sk == ComesfromS:
+    # A template expanded at module toplevel. The wrapper cannot survive
+    # here: its children get split between file scope (global decls) and
+    # the init proc (their initializers and the executable code), so no
+    # single node can span them. Without this unwrap a global-with-call-init
+    # is hoisted whole and codegen emits the initializer as a C
+    # `__attribute__((constructor))`, which runs before NimMain against
+    # zeroed globals (module plugins crashed with 0xC0000005). Dropping the
+    # marker only costs the debug frame for module-init code.
+    n.into:
+      skip n # the origin symbol
+      while n.hasMore:
+        trToplevelItem c, dest, n
+  elif isTopLevelDecl(n):
+    # Pure declarations and compile-time constructs stay at top level:
+    trStmt c, dest, n, TraverseTopLevel
+  else:
+    # Executable code and local vars go into the init proc body:
+    swap dest, c.initBody
+    trStmt c, dest, n, TraverseAll
+    swap dest, c.initBody
+
 proc trToplevel(c: var EContext; dest: var TokenBuf; n: var Cursor) =
   ## Consumes the whole `(stmts …)` node at `n`, including its close.
   n.into:
     while n.hasMore:
-      let sk = n.stmtKind
-      if sk in {GvarS, GletS, TvarS, TletS}:
-        let tag = if sk in {TvarS, TletS}: TvarY else: GvarY
-        if not initHasCall(c, n):
-          # Simple init (literal, nil, etc.): keep at top level.
-          # NIFC can emit "Type var = value;" at C file scope directly.
-          trLocal c, dest, n, tag, TraverseAll, SymId(0)
-        else:
-          # Complex init with function calls: emit a no-init declaration at top
-          # level and place the actual init as an assignment inside the Init proc
-          # body so that any temp variables created by to_stmts remain in scope.
-          let savedN = n
-          trLocal c, dest, n, tag, TraverseSig, SymId(0)
-          var initN = savedN
-          inc initN  # past gvar/glet tag -> at SymbolDef
-          let (initSym, initInfo) = getSymDef(c, initN)
-          skipExportMarker c, initN
-          skip initN  # past pragmas -> at type
-          skip initN  # past type -> at init value
-          swap dest, c.initBody
-          dest.addParLe AsgnS, initInfo
-          dest.addSymUse(initSym, initInfo)
-          trExpr c, dest, initN
-          dest.addParRi()
-          swap dest, c.initBody
-      elif sk == StmtsS:
-        # Nested stmts block: recurse to handle mixed decls and executable code
-        trToplevel c, dest, n
-      elif isTopLevelDecl(n):
-        # Pure declarations and compile-time constructs stay at top level:
-        trStmt c, dest, n, TraverseTopLevel
-      else:
-        # Executable code and local vars go into the init proc body:
-        swap dest, c.initBody
-        trStmt c, dest, n, TraverseAll
-        swap dest, c.initBody
+      trToplevelItem c, dest, n
 
 proc expand*(infile: string; bits: int; bigEndian: bool; flags: set[CheckMode]; isMain: bool; outdir: string; appType = appConsole) =
   let mp = splitModulePath(infile)