Split the dependency collection

Instead of moving the entire sbt-deps phase, this leaves the phase as-is,
and adds new instrumentation in the Inlining phase for the inlined code.

Author: eed3si9n

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -35,6 +35,7 @@ class Compiler {
     List(new TyperPhase) ::         // Compiler frontend: namer, typer
     List(CheckUnused.PostTyper(), CheckShadowing()) :: // Check for unused, shadowed elements
     List(new YCheckPositions) ::    // YCheck positions
+    List(new sbt.ExtractDependencies) :: // Sends information on classes' dependencies to sbt via callbacks
     List(new semanticdb.ExtractSemanticDB.ExtractSemanticInfo) :: // Extract info into .semanticdb files
     List(new PostTyper) ::          // Additional checks and cleanups after type checking
     List(new UnrollDefinitions) ::  // Unroll annotated methods if detected in PostTyper
@@ -47,7 +48,6 @@ class Compiler {
     List(new Pickler) ::            // Generate TASTY info
     List(new sbt.ExtractAPI) ::     // Sends a representation of the API of classes to sbt via callbacks
     List(new Inlining) ::           // Inline and execute macros
-    List(new sbt.ExtractDependencies) :: // Sends information on classes' dependencies to sbt via callbacks
     List(new PostInlining) ::       // Add mirror support for inlined code
     List(new Staging) ::            // Check staging levels and heal staged types
     List(new Splicing) ::           // Replace level 1 splices with holes

compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala

@@ -5,7 +5,7 @@ import scala.language.unsafeNulls
 
 import java.io.File
 import java.nio.file.Path
-import java.util.{Arrays, EnumSet}
+import java.util.EnumSet
 
 import dotty.tools.dotc.ast.tpd
 import dotty.tools.dotc.classpath.FileUtils.{hasClassExtension, hasTastyExtension}
@@ -46,7 +46,6 @@ import scala.compiletime.uninitialized
  *
  *  The following flags affect this phase:
  *   -Yforce-sbt-phases
- *   -Ydump-sbt-inc
  *
  *  @see ExtractAPI
  */
@@ -77,27 +76,6 @@ class ExtractDependencies extends Phase {
     val rec = unit.depRecorder
     val collector = ExtractDependenciesCollector(rec)
     collector.traverse(unit.tpdTree)
-
-    if (ctx.settings.YdumpSbtInc.value) {
-      val deps = rec.foundDeps.iterator.map { case (clazz, found) => s"$clazz: ${found.classesString}" }.toArray[Object]
-      val names = rec.foundDeps.iterator.map { case (clazz, found) => s"$clazz: ${found.namesString}" }.toArray[Object]
-      Arrays.sort(deps)
-      Arrays.sort(names)
-
-      val pw = io.File(unit.source.file.jpath).changeExtension(FileExtension.Inc).toFile.printWriter()
-      // val pw = Console.out
-      try {
-        pw.println("Used Names:")
-        pw.println("===========")
-        names.foreach(pw.println)
-        pw.println()
-        pw.println("Dependencies:")
-        pw.println("=============")
-        deps.foreach(pw.println)
-      } finally pw.close()
-    }
-
-    rec.sendToZinc()
   }
 }
 
@@ -129,6 +107,47 @@ object ExtractDependencies {
     report.error(em"Internal error in the incremental compiler while compiling ${ctx.compilationUnit.source}: $msg", pos)
 }
 
+/** Extract the dependency information of a compilation unit.
+ *
+ *  This extracts the symbol dependencies in the written code.
+ *  There are further extractions performed in the Inlining phase later.
+ *
+ *  To understand why we track the used names see the section "Name hashing
+ *  algorithm" in http://www.scala-sbt.org/0.13/docs/Understanding-Recompilation.html
+ *  To understand why we need to track dependencies introduced by inheritance
+ *  specially, see the subsection "Dependencies introduced by member reference and
+ *  inheritance" in the "Name hashing algorithm" section.
+ */
+private class ExtractDependenciesCollector(rec: DependencyRecorder) extends AbstractExtractDependenciesCollector(rec):
+  import tpd.*
+
+  /** Traverse the tree of a source file and record the dependencies and used names which
+   *  can be retrieved using DependencyRecorder.
+   */
+  override def traverse(tree: Tree)(using Context): Unit =
+    try
+      recordTree(tree)
+      tree match
+        case tree: Inlined if !tree.inlinedFromOuterScope =>
+          // The inlined call is normally ignored by TreeTraverser but we need to
+          // record it as a dependency
+          traverse(tree.call)
+          // traverseChildren(tree)
+        case vd: ValDef if vd.symbol.is(ModuleVal) =>
+          // Don't visit module val
+        case t: Template if t.symbol.owner.is(ModuleClass) =>
+          // Don't visit self type of module class
+          traverse(t.constr)
+          t.parents.foreach(traverse)
+          t.body.foreach(traverse)
+        case _ =>
+          traverseChildren(tree)
+    catch
+      case ex: AssertionError =>
+        println(i"asserted failed while traversing $tree")
+        throw ex
+end ExtractDependenciesCollector
+
 /** Extract the dependency information of a compilation unit.
  *
  *  To understand why we track the used names see the section "Name hashing
@@ -137,7 +156,7 @@ object ExtractDependencies {
  *  specially, see the subsection "Dependencies introduced by member reference and
  *  inheritance" in the "Name hashing algorithm" section.
  */
-private class ExtractDependenciesCollector(rec: DependencyRecorder) extends tpd.TreeTraverser { thisTreeTraverser =>
+trait AbstractExtractDependenciesCollector(rec: DependencyRecorder) extends tpd.TreeTraverser { thisTreeTraverser =>
   import tpd.*
 
   private def addMemberRefDependency(sym: Symbol)(using Context): Unit =
@@ -181,12 +200,8 @@ private class ExtractDependenciesCollector(rec: DependencyRecorder) extends tpd.
       // can happen for constructor proxies. Test case is pos-macros/i13532.
       true
 
-
-  /** Traverse the tree of a source file and record the dependencies and used names which
-   *  can be retrieved using `foundDeps`.
-   */
-  override def traverse(tree: Tree)(using Context): Unit = try {
-    tree match {
+  protected def recordTree(tree: Tree)(using Context): Unit =
+    tree match
       case Match(selector, _) =>
         addPatMatDependency(selector.tpe)
       case Import(expr, selectors) =>
@@ -223,31 +238,7 @@ private class ExtractDependenciesCollector(rec: DependencyRecorder) extends tpd.
         addInheritanceDependencies(t)
       case t: Template =>
         addInheritanceDependencies(t)
-      case _ =>
-    }
-
-    tree match {
-      case tree: Inlined =>
-        // The inlined call is normally ignored by TreeTraverser but we need to
-        // record it as a dependency
-        if !tree.inlinedFromOuterScope then
-          traverse(tree.call)
-        traverseChildren(tree)
-      case vd: ValDef if vd.symbol.is(ModuleVal) =>
-        // Don't visit module val
-      case t: Template if t.symbol.owner.is(ModuleClass) =>
-        // Don't visit self type of module class
-        traverse(t.constr)
-        t.parents.foreach(traverse)
-        t.body.foreach(traverse)
-      case _ =>
-        traverseChildren(tree)
-    }
-  } catch {
-    case ex: AssertionError =>
-      println(i"asserted failed while traversing $tree")
-      throw ex
-  }
+      case _ => ()
 
   /**Reused EqHashSet, safe to use as each TypeDependencyTraverser is used atomically
    * Avoid cycles by remembering both the types (testcase:

compiler/src/dotty/tools/dotc/transform/Inlining.scala

@@ -1,6 +1,10 @@
 package dotty.tools.dotc
 package transform
 
+
+import java.util.Arrays
+
+import dotty.tools.io
 import ast.tpd
 import ast.Trees.*
 import ast.TreeMapWithTrackedStats
@@ -15,12 +19,20 @@ import DenotTransformers.IdentityDenotTransformer
 import MacroAnnotations.hasMacroAnnotation
 import inlines.Inlines
 import quoted.*
+import sbt.{ AbstractExtractDependenciesCollector, DependencyRecorder }
 import staging.StagingLevel
 import util.Property
 
 import scala.collection.mutable
-
-/** Inlines all calls to inline methods that are not in an inline method or a quote */
+import scala.io.Codec
+
+/**
+ * Inlines all calls to inline methods that are not in an inline method or a quote.
+ *
+ * The following flags affect this phase:
+ *   -Ydump-sbt-inc
+ *
+ */
 class Inlining extends MacroTransform, IdentityDenotTransformer {
   self =>
 
@@ -35,8 +47,33 @@ class Inlining extends MacroTransform, IdentityDenotTransformer {
   override def changesMembers: Boolean = true
 
   override def run(using Context): Unit =
-    if ctx.compilationUnit.needsInlining || ctx.compilationUnit.hasMacroAnnotations then
+    val unit = ctx.compilationUnit
+    val rec = ctx.compilationUnit.depRecorder
+    if unit.needsInlining || unit.hasMacroAnnotations then
       super.run
+    rec.sendToZinc()
+    ctx.run.nn.asyncTasty.foreach(_.signalDepsExtractionComplete())
+
+    if ctx.settings.YdumpSbtInc.value then
+      val deps = rec.foundDeps.iterator.map { case (clazz, found) => s"$clazz: ${found.classesString}" }.toArray[Object]
+      val names = rec.foundDeps.iterator.map { case (clazz, found) => s"$clazz: ${found.namesString}" }.toArray[Object]
+      Arrays.sort(deps)
+      Arrays.sort(names)
+
+      unit.source.file.jpath match
+        case jpath: io.JPath =>
+          val pw = io.File(jpath)(using Codec.UTF8).changeExtension(io.FileExtension.Inc).toFile.printWriter()
+          // val pw = Console.out
+          try
+            pw.println("Used Names:")
+            pw.println("===========")
+            names.foreach(pw.println)
+            pw.println()
+            pw.println("Dependencies:")
+            pw.println("=============")
+            deps.foreach(pw.println)
+          finally pw.close()
+        case null => ()
 
   override def checkPostCondition(tree: Tree)(using Context): Unit =
     tree match {
@@ -54,18 +91,49 @@ class Inlining extends MacroTransform, IdentityDenotTransformer {
 
   def newTransformer(using Context): Transformer = new Transformer {
     override def transform(tree: tpd.Tree)(using Context): tpd.Tree =
-      InliningTreeMap().transform(tree)
+      val rec = ctx.compilationUnit.depRecorder
+      val collector = ExtractInlineDependenciesCollector(rec)
+      InliningTreeMap(collector).transform(tree)
   }
 
-  private class InliningTreeMap extends TreeMapWithTrackedStats {
+  private class ExtractInlineDependenciesCollector(rec: DependencyRecorder) extends AbstractExtractDependenciesCollector(rec):
+    /** Traverse the tree of a source file and record the dependencies and used names which
+     *  can be retrieved using `rec`.
+     */
+    override def traverse(tree: Tree)(using Context): Unit =
+      recordTree(tree)
+      traverseChildren(tree)
+  end ExtractInlineDependenciesCollector
+
+  private class InliningTreeMap(collector: ExtractInlineDependenciesCollector) extends TreeMapWithTrackedStats {
 
     /** List of top level classes added by macro annotation in a package object.
      *  These are added to the PackageDef that owns this particular package object.
      */
     private val newTopClasses = MutableSymbolMap[mutable.ListBuffer[Tree]]()
 
+    val inlineFinder = new tpd.TreeTraverser:
+      override def traverse(tree: Tree)(using Context): Unit =
+        try
+          tree match
+            case tree: Inlined =>
+              collector.traverse(tree)
+            case vd: ValDef if vd.symbol.is(ModuleVal) =>
+              // Don't visit module val
+            case t: Template if t.symbol.owner.is(ModuleClass) =>
+              // Don't visit self type of module class
+              traverse(t.constr)
+              t.parents.foreach(traverse)
+              t.body.foreach(traverse)
+            case _ =>
+              traverseChildren(tree)
+        catch
+          case ex: AssertionError =>
+            println(i"asserted failed while traversing $tree")
+            throw ex
+
     override def transform(tree: Tree)(using Context): Tree = {
-      tree match
+      val result = tree match
         case tree: MemberDef =>
           // Fetch the latest tracked tree (It might have already been transformed by its companion)
           transformMemberDef(getTracked(tree.symbol).getOrElse(tree))
@@ -93,6 +161,8 @@ class Inlining extends MacroTransform, IdentityDenotTransformer {
                 if tree1.tpe.isError then tree1
                 else Inlines.inlineCall(tree1)
           else super.transform(tree)
+      inlineFinder.traverse(result)
+      result
     }
 
     private def transformMemberDef(tree: MemberDef)(using Context) : Tree =

compiler/src/dotty/tools/dotc/transform/Pickler.scala

@@ -74,10 +74,14 @@ object Pickler {
     private val asyncAPIComplete =
       if incCallback == null then Promise.successful(Signal.Done) // no need to wait for API completion
       else Promise[Signal]()
+    private val asyncDepsExtractionComplete =
+      if incCallback == null then Promise.successful(Signal.Done)
+      else Promise[Signal]()
 
     private val backendFuture: StdFuture[Option[BufferingReporter]] =
       val asyncState = asyncTastyWritten.future
         .zipWith(asyncAPIComplete.future)((state, api) => state.filterNot(_ => api == Signal.Cancelled))
+        .zipWith(asyncDepsExtractionComplete.future)((state, deps) => state.filterNot(_ => deps == Signal.Cancelled))
       asyncState.map: optState =>
         optState.flatMap: state =>
           if incCallback != null && state.done && !state.hasErrors then
@@ -92,6 +96,10 @@ object Pickler {
       if incCallback != null then
         asyncAPIComplete.trySuccess(Signal.Done)
 
+    def signalDepsExtractionComplete(): Unit =
+      if incCallback != null then
+        asyncDepsExtractionComplete.trySuccess(Signal.Done)
+
     /** should only be called once */
     def signalAsyncTastyWritten()(using ctx: ReadOnlyContext): Unit =
       val done = !ctx.run.suspendedAtTyperPhase