Address feedback. Adjust semantics.

Now in backend we block on the composition of the Zinc API callbacks
and tasty being written.

We also now conditionally signal that we are "done" writing tasty,
based on if any units were suspended. This works in line with the
Zinc default, which will ignore the early output anyway under the
presence of macros (user can override this). Also move async tasty
holder to the Run, as it is now context dependent on suspending.

TODO: give the user an option to optimise performance by preventing definition
of such problematic macros (which would also avoid suspensions).

Author: bishabosha

compiler/src/dotty/tools/backend/jvm/ClassfileWriters.scala

@@ -20,7 +20,11 @@ import dotty.tools.io.JarArchive
 
 import scala.language.unsafeNulls
 
-
+/** !!! This file is now copied in `dotty.tools.io.FileWriters` in a more general way that does not rely upon
+ * `PostProcessorFrontendAccess`, this should probably be changed to wrap that class instead.
+ *
+ * Until then, any changes to this file should be copied to `dotty.tools.io.FileWriters` as well.
+ */
 class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
   type NullableFile =  AbstractFile | Null
   import frontendAccess.{compilerSettings, backendReporting}

compiler/src/dotty/tools/backend/jvm/GenBCode.scala

@@ -12,7 +12,7 @@ import scala.collection.mutable
 import scala.compiletime.uninitialized
 import java.util.concurrent.TimeoutException
 
-import scala.concurrent.duration.given
+import scala.concurrent.duration.Duration
 import scala.concurrent.Await
 
 class GenBCode extends Phase { self =>
@@ -94,20 +94,15 @@ class GenBCode extends Phase { self =>
     try
       val result = super.runOn(units)
       generatedClassHandler.complete()
-      for holder <- ctx.asyncTastyPromise do
-        try
-          val asyncState = Await.result(holder.promise.future, 5.seconds)
-          for reporter <- asyncState.pending do
-            reporter.relayReports(frontendAccess.backendReporting)
-        catch
-          case _: TimeoutException =>
-            report.error(
-              """Timeout (5s) in backend while waiting for async writing of TASTy files to -Yearly-tasty-output,
-                |  this may be a bug in the compiler.
-                |
-                |Alternatively consider turning off pipelining for this project.""".stripMargin
-            )
-      end for
+      try
+        for
+          async <- ctx.run.nn.asyncTasty
+          bufferedReporter <- async.sync()
+        do
+          bufferedReporter.relayReports(frontendAccess.backendReporting)
+      catch
+        case ex: Exception =>
+          report.error(s"exception from future: $ex, (${Option(ex.getCause())})")
       result
     finally
       // frontendAccess and postProcessor are created lazilly, clean them up only if they were initialized

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -98,11 +98,15 @@ class CompilationUnit protected (val source: SourceFile, val info: CompilationUn
     depRecorder.clear()
     if !suspended then
       suspended = true
-      ctx.run.nn.suspendedUnits += this
+      val currRun = ctx.run.nn
+      currRun.suspendedUnits += this
+      val isInliningPhase = ctx.phase == Phases.inliningPhase
       if ctx.settings.XprintSuspension.value then
-        ctx.run.nn.suspendedHints += (this -> hint)
-      if ctx.phase == Phases.inliningPhase then
+        currRun.suspendedHints += (this -> (hint, isInliningPhase))
+      if isInliningPhase then
         suspendedAtInliningPhase = true
+      else
+        currRun.suspendedAtTyperPhase = true
     throw CompilationUnit.SuspendException()
 
   private var myAssignmentSpans: Map[Int, List[Span]] | Null = null

compiler/src/dotty/tools/dotc/Driver.scala

@@ -54,10 +54,10 @@ class Driver {
       if (ctx.settings.XprintSuspension.value)
         val suspendedHints = run.suspendedHints.toList
         report.echo(i"compiling suspended $suspendedUnits%, %")
-        for (unit, hint) <- suspendedHints do
-          report.echo(s"  $unit: $hint")
+        for (unit, (hint, atInlining)) <- suspendedHints do
+          report.echo(s"  $unit at ${if atInlining then "inlining" else "typer"}: $hint")
       val run1 = compiler.newRun
-      run1.compileSuspendedUnits(suspendedUnits)
+      run1.compileSuspendedUnits(suspendedUnits, !run.suspendedAtTyperPhase)
       finish(compiler, run1)(using MacroClassLoader.init(ctx.fresh))
 
   protected def initCtx: Context = (new ContextBase).initialCtx
@@ -66,13 +66,6 @@ class Driver {
 
   protected def command: CompilerCommand = ScalacCommand
 
-  private def setupAsyncTasty(ictx: FreshContext): Unit = inContext(ictx):
-    ictx.settings.YearlyTastyOutput.value match
-      case earlyOut if earlyOut.isDirectory && earlyOut.exists =>
-        ictx.setInitialAsyncTasty()
-      case _ =>
-        () // do nothing
-
   /** Setup context with initialized settings from CLI arguments, then check if there are any settings that
    *  would change the default behaviour of the compiler.
    *
@@ -89,7 +82,6 @@ class Driver {
     Positioned.init(using ictx)
 
     inContext(ictx) {
-      setupAsyncTasty(ictx)
       if !ctx.settings.YdropComments.value || ctx.settings.YreadComments.value then
         ictx.setProperty(ContextDoc, new ContextDocstrings)
       val fileNamesOrNone = command.checkUsage(summary, sourcesRequired)(using ctx.settings)(using ctx.settingsState)

compiler/src/dotty/tools/dotc/Run.scala

@@ -37,6 +37,7 @@ import scala.io.Codec
 import Run.Progress
 import scala.compiletime.uninitialized
 import dotty.tools.dotc.transform.MegaPhase
+import dotty.tools.dotc.transform.Pickler.AsyncTastyHolder
 
 /** A compiler run. Exports various methods to compile source files */
 class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with ConstraintRunInfo {
@@ -130,7 +131,10 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
     myUnits = us
 
   var suspendedUnits: mutable.ListBuffer[CompilationUnit] = mutable.ListBuffer()
-  var suspendedHints: mutable.Map[CompilationUnit, String] = mutable.HashMap()
+  var suspendedHints: mutable.Map[CompilationUnit, (String, Boolean)] = mutable.HashMap()
+
+  /** Were any units suspended in the typer phase? if so then pipeline tasty can not complete. */
+  var suspendedAtTyperPhase: Boolean = false
 
   def checkSuspendedUnits(newUnits: List[CompilationUnit])(using Context): Unit =
     if newUnits.isEmpty && suspendedUnits.nonEmpty && !ctx.reporter.errorsReported then
@@ -231,6 +235,22 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
         if !progress.isCancelled() then
           progress.tickSubphase()
 
+  /** if true, then we are done writing pipelined TASTy files (i.e. finished in a previous run.) */
+  private var myAsyncTastyWritten = false
+
+  private var _asyncTasty: Option[AsyncTastyHolder] = None
+
+  /** populated when this run needs to write pipeline TASTy files. */
+  def asyncTasty: Option[AsyncTastyHolder] = _asyncTasty
+
+  private def initializeAsyncTasty()(using Context): () => Unit =
+    // should we provide a custom ExecutionContext?
+    // currently it is just used to call the `apiPhaseCompleted` and `dependencyPhaseCompleted` callbacks in Zinc
+    import scala.concurrent.ExecutionContext.Implicits.global
+    val async = AsyncTastyHolder.init
+    _asyncTasty = Some(async)
+    () => async.cancel()
+
   /** Will be set to true if any of the compiled compilation units contains
    *  a pureFunctions language import.
    */
@@ -348,7 +368,14 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
       runCtx.setProperty(CyclicReference.Trace, new CyclicReference.Trace())
     runCtx.withProgressCallback: cb =>
       _progress = Progress(cb, this, fusedPhases.map(_.traversals).sum)
+    val cancelAsyncTasty: () => Unit =
+      if !myAsyncTastyWritten && Phases.picklerPhase.exists && !ctx.settings.YearlyTastyOutput.isDefault then
+        initializeAsyncTasty()
+      else () => {}
+
     runPhases(allPhases = fusedPhases)(using runCtx)
+    cancelAsyncTasty()
+
     ctx.reporter.finalizeReporting()
     if (!ctx.reporter.hasErrors)
       Rewrites.writeBack()
@@ -365,9 +392,12 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
   /** Is this run started via a compilingSuspended? */
   def isCompilingSuspended: Boolean = myCompilingSuspended
 
-  /** Compile units `us` which were suspended in a previous run */
-  def compileSuspendedUnits(us: List[CompilationUnit]): Unit =
+  /** Compile units `us` which were suspended in a previous run,
+   *  also signal if all necessary async tasty files were written in a previous run.
+   */
+  def compileSuspendedUnits(us: List[CompilationUnit], asyncTastyWritten: Boolean): Unit =
     myCompilingSuspended = true
+    myAsyncTastyWritten = asyncTastyWritten
     for unit <- us do unit.suspended = false
     compileUnits(us)
 

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -31,11 +31,9 @@ import StdNames.nme
 import compiletime.uninitialized
 
 import scala.annotation.internal.sharable
-import scala.concurrent.Promise
 
 import DenotTransformers.DenotTransformer
 import dotty.tools.dotc.profile.Profiler
-import dotty.tools.dotc.transform.Pickler.AsyncTastyHolder
 import dotty.tools.dotc.sbt.interfaces.{IncrementalCallback, ProgressCallback}
 import util.Property.Key
 import util.Store
@@ -56,9 +54,8 @@ object Contexts {
   private val (importInfoLoc,        store9) = store8.newLocation[ImportInfo | Null]()
   private val (typeAssignerLoc,     store10) = store9.newLocation[TypeAssigner](TypeAssigner)
   private val (progressCallbackLoc, store11) = store10.newLocation[ProgressCallback | Null]()
-  private val (tastyPromiseLoc,     store12) = store11.newLocation[Option[AsyncTastyHolder]](None)
 
-  private val initialStore = store12
+  private val initialStore = store11
 
   /** The current context */
   inline def ctx(using ctx: Context): Context = ctx
@@ -200,8 +197,6 @@ object Contexts {
     /** The current settings values */
     def settingsState: SettingsState = store(settingsStateLoc)
 
-    def asyncTastyPromise: Option[AsyncTastyHolder] = store(tastyPromiseLoc)
-
     /** The current compilation unit */
     def compilationUnit: CompilationUnit = store(compilationUnitLoc)
 
@@ -690,9 +685,6 @@ object Contexts {
       updateStore(compilationUnitLoc, compilationUnit)
     }
 
-    def setInitialAsyncTasty(): this.type =
-      assert(store(tastyPromiseLoc) == None, "trying to set async tasty promise twice!")
-      updateStore(tastyPromiseLoc, Some(AsyncTastyHolder(settings.YearlyTastyOutput.value, Promise())))
 
     def setCompilerCallback(callback: CompilerCallback): this.type = updateStore(compilerCallbackLoc, callback)
     def setIncCallback(callback: IncrementalCallback): this.type = updateStore(incCallbackLoc, callback)

compiler/src/dotty/tools/dotc/sbt/ExtractAPI.scala

@@ -90,14 +90,7 @@ class ExtractAPI extends Phase {
       val sourceFile = cls.source
       if sourceFile.exists && cls.isDefinedInCurrentRun then
         recordNonLocalClass(cls, sourceFile, cb)
-    for holder <- ctx.asyncTastyPromise do
-      import scala.concurrent.ExecutionContext.Implicits.global
-      // do not expect to be completed with failure
-      holder.promise.future.foreach: state =>
-        if !state.hasErrors then
-          // We also await the promise at GenBCode to emit warnings/errors
-          cb.apiPhaseCompleted()
-          cb.dependencyPhaseCompleted()
+    ctx.run.nn.asyncTasty.foreach(_.signalAPIComplete())
 
   private def recordNonLocalClass(cls: Symbol, sourceFile: SourceFile, cb: interfaces.IncrementalCallback)(using Context): Unit =
     def registerProductNames(fullClassName: String, binaryClassName: String) =

compiler/src/dotty/tools/dotc/sbt/package.scala

@@ -6,10 +6,29 @@ import dotty.tools.dotc.core.NameOps.stripModuleClassSuffix
 import dotty.tools.dotc.core.Names.Name
 import dotty.tools.dotc.core.Names.termName
 
+import interfaces.IncrementalCallback
+import dotty.tools.io.FileWriters.BufferingReporter
+import dotty.tools.dotc.core.Decorators.em
+
+import scala.util.chaining.given
+import scala.util.control.NonFatal
+
 inline val TermNameHash = 1987 // 300th prime
 inline val TypeNameHash = 1993 // 301st prime
 inline val InlineParamHash = 1997 // 302nd prime
 
+def asyncZincPhasesCompleted(cb: IncrementalCallback, pending: Option[BufferingReporter]): Option[BufferingReporter] =
+  val zincReporter = pending match
+    case Some(buffered) => buffered
+    case None => BufferingReporter()
+  try
+    cb.apiPhaseCompleted()
+    cb.dependencyPhaseCompleted()
+  catch
+    case NonFatal(t) =>
+      zincReporter.exception(em"signaling API and Dependencies phases completion", t)
+  if zincReporter.hasErrors then Some(zincReporter) else None
+
 extension (sym: Symbol)
 
   /** Mangle a JVM symbol name in a format better suited for internal uses by sbt.