fixes #13736; Compiler crash with auto return type and recursion

compiler/sem.nim

@@ -108,6 +108,16 @@ proc fitNodePostMatch(c: PContext, formal: PType, arg: PNode): PNode =
     markUsed(c, a.info, a[0].sym)
 
 
+template isAutoReturnType(t: PType): bool =
+  # `auto` return types are copied and marked so they are not generic params.
+  t.kind == tyAnything and tfRetType in t.flags
+
+template isUnresolvedAutoReturnType(c: PContext; t: PType): bool =
+  # During return-type inference a recursive call has the routine's exact
+  # `auto` placeholder type. It contributes no type information of its own.
+  c.p != nil and c.p.owner != nil and c.p.owner.typ != nil and
+    c.p.owner.typ.returnType == t and isAutoReturnType(t)
+
 proc fitNode(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode =
   if arg.typ.isNil:
     localError(c.config, arg.info, "expression has no type: " &
@@ -125,6 +135,10 @@ proc fitNode(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode =
       if sameType(ch.typ.skipTypes({tyVar, tyLent}), formal):
         return ch
     typeMismatch(c.config, info, formal, arg.typ, arg)
+  elif isUnresolvedAutoReturnType(c, arg.typ):
+    # A concrete sibling branch supplies the missing type for this branch.
+    result = arg
+    changeType(c, result, formal, check=true)
   else:
     result = indexTypesMatch(c, formal, arg.typ, arg)
     if result == nil:
@@ -158,8 +172,10 @@ proc commonType*(c: PContext; x, y: PType): PType =
   var a = skipTypes(x, {tyGenericInst, tyAlias, tySink})
   var b = skipTypes(y, {tyGenericInst, tyAlias, tySink})
   result = x
-  if a.kind in {tyUntyped, tyNil}: result = y
-  elif b.kind in {tyUntyped, tyNil}: result = x
+  # Recursive calls cannot contribute to their own `auto` return type, so let
+  # the other branch determine the common type when it has concrete evidence.
+  if a.kind in {tyUntyped, tyNil} or isUnresolvedAutoReturnType(c, a): result = y
+  elif b.kind in {tyUntyped, tyNil} or isUnresolvedAutoReturnType(c, b): result = x
   elif a.kind == tyTyped: result = a
   elif b.kind == tyTyped: result = b
   elif a.kind == tyTypeDesc:

compiler/semcall.nim

@@ -924,6 +924,11 @@ proc semResolvedCall(c: PContext, x: var TCandidate,
   result[0] = newSymNode(finalCallee, getCallLineInfo(result[0]))
   if finalCallee.magic notin {mArrGet, mArrPut}:
     result.typ = finalCallee.typ.returnType
+    # Remember that this body contains a self-call still sharing its unresolved
+    # `auto` placeholder; a later concrete return must resolve that placeholder.
+    if c.p != nil and result.typ != nil and finalCallee == c.p.owner and
+        isAutoReturnType(result.typ):
+      c.p.hasUnresolvedAutoCall = true
   updateDefaultParams(c, result)
 
 proc canDeref(n: PNode): bool {.inline.} =

compiler/semdata.nim

@@ -43,6 +43,7 @@ type
     mapping*: SymMapping
     caseContext*: seq[tuple[n: PNode, idx: int]]
     localBindStmts*: seq[PNode]
+    hasUnresolvedAutoCall*: bool # a self-call still uses the `auto` return placeholder
 
   TMatchedConcept* = object
     candidateType*: PType

compiler/semexprs.nim

@@ -2125,6 +2125,15 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
           internalAssert c.config, c.p.resultSym != nil
           # Make sure the type is valid for the result variable
           typeAllowedCheck(c, n.info, rhsTyp, skResult)
+          # Earlier self-calls retain the old placeholder pointer. Resolve it
+          # in place as an alias before the routine switches to the concrete
+          # type, so those already-typed calls see the inferred type too.
+          if c.p.hasUnresolvedAutoCall and not rhsTyp.isMetaType and
+              isAutoReturnType(lhs.sym.typ):
+            let resolved = newTypeS(tyAlias, c)
+            rawAddSon(resolved, rhsTyp)
+            assignType(lhs.sym.typ, resolved)
+            c.p.hasUnresolvedAutoCall = false
           lhs.typ = rhsTyp
           c.p.resultSym.typ = rhsTyp
           c.p.owner.typ.setReturnType rhsTyp
@@ -2196,7 +2205,11 @@ proc semProcBody(c: PContext, n: PNode; expectedType: PType = nil): PNode =
         " flags=", c.p.resultSym.typ.flags,
         " uid=", c.p.resultSym.typ.uniqueId.module, ".", c.p.resultSym.typ.uniqueId.item,
         " state=", c.p.resultSym.typ.state
-    if isEmptyType(result.typ):
+    # With no concrete return, the recursive placeholder is still circular.
+    if c.p.hasUnresolvedAutoCall:
+      localError(c.config, c.p.resultSym.info, errCannotInferReturnType %
+        c.p.owner.name.s)
+    elif isEmptyType(result.typ):
       # we inferred a 'void' return type:
       c.p.resultSym.typ = errorType(c)
       c.p.owner.typ.setReturnType nil

tests/errmsgs/t13736.nim

@@ -0,0 +1,9 @@
+discard """
+  errormsg: "cannot infer the return type of 'foo'"
+  line: 6
+"""
+
+proc foo(n: int): auto =
+  return foo(n + 1)
+
+discard foo(0)

tests/errmsgs/t13736_expression.nim

@@ -0,0 +1,12 @@
+discard """
+  errormsg: "cannot infer the return type of 'foo'"
+  line: 6
+"""
+
+proc foo(n: int): auto =
+  if n > 0:
+    foo(n - 1)
+  else:
+    foo(n + 1)
+
+discard foo(1)

tests/errmsgs/t13736_generic.nim

@@ -0,0 +1,9 @@
+discard """
+  errormsg: "cannot infer the return type of 'foo'"
+  line: 6
+"""
+
+proc foo[T](x: T): auto =
+  foo(x)
+
+discard foo(1)

tests/misc/t13736.nim

@@ -0,0 +1,46 @@
+
+
+proc byReturn(n: int): auto =
+  if n < 5:
+    return byReturn(n + 1)
+  else:
+    return 9
+
+proc byResult(n: int): auto =
+  if n < 5:
+    result = byResult(n + 1)
+  else:
+    result = 9
+
+proc byExpression(n: int): auto =
+  if n < 5:
+    byExpression(n + 1)
+  else:
+    9
+
+proc generic[T](x: T; n: int): auto =
+  if n < 5:
+    return generic(x, n + 1)
+  else:
+    return x
+
+proc concreteFirst(n: int): auto =
+  if n >= 5:
+    return 9
+  else:
+    return concreteFirst(n + 1)
+
+proc multipleRecursiveBranches(n: int): auto =
+  if n < 0:
+    return multipleRecursiveBranches(n + 1)
+  elif n < 5:
+    return multipleRecursiveBranches(n + 1)
+  else:
+    return 9
+
+doAssert byReturn(3) == 9
+doAssert byResult(3) == 9
+doAssert byExpression(3) == 9
+doAssert generic("ok", 3) == "ok"
+doAssert concreteFirst(3) == 9
+doAssert multipleRecursiveBranches(-1) == 9