spawn Nim compilation

compiler/commands.nim

@@ -529,6 +529,13 @@ proc setCmd*(conf: ConfigRef, cmd: Command) =
     conf.globalOptions.incl optCompress
   else: discard
 
+  # Enable spawnCodegen by default for C/C++ compilation commands to reclaim
+  # memory before C compilation.
+  # In theory, `optCRun` would benefit from this treatment as well but this gets
+  # complex with its command line hashing
+  if cmd in {cmdCompileToC, cmdCompileToCpp}:
+    conf.globalOptions.incl optSpawnCodegen
+
 proc setCommandEarly*(conf: ConfigRef, command: string) =
   conf.command = command
   setCmd(conf, command.parseCommand)
@@ -1112,6 +1119,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if switch.normalize == "gendeps": deprecatedAlias(switch, "genscript")
     processOnOffSwitchG(conf, {optGenScript}, arg, pass, info)
     processOnOffSwitchG(conf, {optCompileOnly}, arg, pass, info)
+  of "spawncodegen":
+    processOnOffSwitchG(conf, {optSpawnCodegen}, arg, pass, info)
   of "gencdeps":
     processOnOffSwitchG(conf, {optGenCDeps}, arg, pass, info)
   of "colors": processOnOffSwitchG(conf, {optUseColors}, arg, pass, info)

compiler/extccomp.nim

@@ -1163,6 +1163,55 @@ proc runJsonBuildInstructions*(conf: ConfigRef; jsonFile: AbsoluteFile) =
   preventLinkCmdMaxCmdLen(conf, bcache.linkcmd)
   for cmd in bcache.extraCmds: execExternalProgram(conf, cmd, hintExecuting)
 
+proc spawnCodegenSubprocess*(conf: ConfigRef) =
+  ## Spawns a separate nim process with --compileOnly to perform
+  ## Nim-to-C code generation, then runs the C compile/link steps from the
+  ## generated JSON build instructions. This reclaims the Nim compiler's memory
+  ## before proceeding with C compilation.
+
+  # The subprocess args consist of the existing args and options up to the
+  # project file with `--compileOnly` injected first - anything after the project
+  # file is meant for running the project (`-r`) so we should have exactly two
+  # non-option arguments.
+  # We also disable the conf hint since it would otherwise show twice as the
+  # config files get parsed by both processes.
+  var subArgs = @["--compileOnly", "--hint[Conf]:off"]
+  var projectFileAdded = false
+  var commandAdded = false
+  for a in os.commandLineParams():
+    if a.len == 0:
+      continue
+
+    if a notin ["-r", "--run"]:
+      subArgs.add a
+
+    if a[0] != '-':
+      if commandAdded:
+        projectFileAdded = true
+        break
+      else:
+        commandAdded = true
+
+  doAssert projectFileAdded, "Could not find project file in command line, bug?"
+
+  # Spawn subprocess - the subprocess generates C files + JSON build instructions
+  let nimExe = getAppFilename()
+  try:
+    let p = startProcess(nimExe, args = subArgs, options = {poParentStreams})
+    let exitCode = p.waitForExit()
+    p.close()
+    if exitCode != 0:
+      # We assume the internal compiler has printed its own messages - the test
+      # suite depends on nothing being printed here
+      inc conf.errorCounter
+      return
+  except CatchableError as e:
+    rawMessage(conf, errGenerated, "execution of codegen failed: '$1'" %
+      [e.msg])
+    return
+
+  runJsonBuildInstructions(conf, conf.jsonBuildInstructionsFile)
+
 proc genMappingFiles(conf: ConfigRef; list: CfileList): Rope =
   result = ""
   for it in list:

compiler/main.nim

@@ -155,6 +155,17 @@ proc commandCompileToC(graph: ModuleGraph) =
         graph.config.notes = graph.config.mainPackageNotes
         return
 
+  # Spawn a separate nim process for code generation to reclaim memory
+  # before C compilation. The subprocess runs with --compileOnly, generates the
+  # C code and a JSON build script then executes the build script to build the
+  # final output.
+  # optHotCodeReloading is mostly broken in general
+  # optUseNimcache requires changes to how command lines are hashed
+  if {optSpawnCodegen, optCompileOnly, optHotCodeReloading, optUseNimcache} *
+      conf.globalOptions == {optSpawnCodegen}:
+    extccomp.spawnCodegenSubprocess(conf)
+    return # Subprocess handled everything; skip in-process compilation
+
   if not extccomp.ccHasSaneOverflow(conf):
     conf.symbols.defineSymbol("nimEmulateOverflowChecks")
 

compiler/msgs.nim

@@ -711,6 +711,9 @@ proc genSuccessX*(conf: ConfigRef) =
   var build = ""
   var flags = ""
   const debugModeHints = "none (DEBUG BUILD, `-d:release` generates faster code)"
+  if optCompileOnly in conf.globalOptions:
+    build = "codegen "
+
   if conf.cmd in cmdBackends:
     if conf.backend != backendJs:
       build.add "mm: $#; " % $conf.selectedGC
@@ -737,7 +740,7 @@ proc genSuccessX*(conf: ConfigRef) =
     # xxx honor conf.filenameOption more accurately
   var output: string
   if optCompileOnly in conf.globalOptions and conf.cmd != cmdJsonscript:
-    output = $conf.jsonBuildFile
+    output = $conf.getNimcacheDir()
   elif conf.outFile.isEmpty and conf.cmd notin {cmdJsonscript} + cmdDocLike + cmdBackends:
     # for some cmd we expect a valid absOutFile
     output = "unknownOutput"

compiler/options.nim

@@ -127,6 +127,8 @@ type                          # please make sure we have under 32 options
                               # string/seq implementation based on destructors
     optTinyRtti               # active if we use the new "tiny RTTI"
                               # implementation
+    optSpawnCodegen           # spawn a separate nim process for codegen
+                              # to reclaim memory before C compilation
     optOwnedRefs              # active if the Nim compiler knows about 'owned'.
     optMultiMethods
     optBenchmarkVM            # Enables cpuTime() in the VM

doc/advopt.txt

@@ -185,3 +185,4 @@ Advanced options:
                             the JavaScript backend (default: on)
   --nimBasePattern:nimbase.h
                             allows to specify a custom pattern for `nimbase.h`
+  --spawncodegen:on|off     spawn a separate process for the codegen phase