Merge pull request #10670 from scalacenter/tasty/support-scala-3.4

TASTy Reader: support Scala 3.4 [ci: last-only]

project/DottySupport.scala

@@ -12,7 +12,7 @@ import sbt.librarymanagement.{
   * Settings to support validation of TastyUnpickler against the release of dotty with the matching TASTy version
   */
 object TastySupport {
-  val supportedTASTyRelease = "3.3.1"
+  val supportedTASTyRelease = "3.4.0-RC4" // TASTY: 28.4-experimental-1 (preparing for final release 28.4)
   val scala3Compiler = "org.scala-lang" % "scala3-compiler_3" % supportedTASTyRelease
   val scala3Library = "org.scala-lang" % "scala3-library_3" % supportedTASTyRelease
 

src/compiler/scala/tools/nsc/classpath/DirectoryClassPath.scala

@@ -41,12 +41,13 @@ trait DirectoryLookup[FileEntryType <: ClassRepresentation] extends EfficientCla
   protected def emptyFiles: Array[F] // avoids reifying ClassTag[F]
   protected def getSubDir(dirName: String): Option[F]
   protected def listChildren(dir: F, filter: Option[F => Boolean] = None): Array[F]
+  protected def hasChild(dir: F, name: String): Boolean
   protected def getName(f: F): String
   protected def toAbstractFile(f: F): AbstractFile
   protected def isPackage(f: F): Boolean
 
   protected def createFileEntry(file: AbstractFile): FileEntryType
-  protected def isMatchingFile(f: F): Boolean
+  protected def isMatchingFile(f: F, siblingExists: String => Boolean): Boolean
 
   private def getDirectory(forPackage: PackageName): Option[F] = {
     if (forPackage.isRoot) {
@@ -73,7 +74,9 @@ trait DirectoryLookup[FileEntryType <: ClassRepresentation] extends EfficientCla
     val dirForPackage = getDirectory(inPackage)
     val files: Array[F] = dirForPackage match {
       case None => emptyFiles
-      case Some(directory) => listChildren(directory, Some(isMatchingFile))
+      case Some(directory) =>
+        val hasCh = hasChild(directory, _)
+        listChildren(directory, Some(f => isMatchingFile(f, hasCh)))
     }
     files.iterator.map(f => createFileEntry(toAbstractFile(f))).toSeq
   }
@@ -83,10 +86,11 @@ trait DirectoryLookup[FileEntryType <: ClassRepresentation] extends EfficientCla
     dirForPackage match {
       case None =>
       case Some(directory) =>
+        val hasCh = hasChild(directory, _)
         for (file <- listChildren(directory)) {
           if (isPackage(file))
             onPackageEntry(PackageEntryImpl(inPackage.entryName(getName(file))))
-          else if (isMatchingFile(file))
+          else if (isMatchingFile(file, hasCh))
             onClassesAndSources(createFileEntry(toAbstractFile(file)))
         }
     }
@@ -121,6 +125,7 @@ trait JFileDirectoryLookup[FileEntryType <: ClassRepresentation] extends Directo
     java.util.Arrays.sort(listing, (o1: File, o2: File) => o1.getName.compareTo(o2.getName))
     listing
   }
+  protected def hasChild(dir: File, name: String): Boolean = new File(dir, name).isFile
   protected def getName(f: File): String = f.getName
   protected def toAbstractFile(f: File): AbstractFile = new PlainFile(new scala.reflect.io.File(f))
   protected def isPackage(f: File): Boolean = f.isPackage
@@ -328,7 +333,8 @@ case class DirectoryClassPath(dir: File) extends JFileDirectoryLookup[ClassFileE
   }
 
   protected def createFileEntry(file: AbstractFile): ClassFileEntryImpl = ClassFileEntryImpl(file)
-  protected def isMatchingFile(f: File): Boolean = f.isClass
+  protected def isMatchingFile(f: File, siblingExists: String => Boolean): Boolean =
+    f.isClass && !(f.getName.endsWith(".class") && siblingExists(f.getName.dropRight(6) + ".tasty"))
 
   private[nsc] def classes(inPackage: PackageName): Seq[ClassFileEntry] = files(inPackage)
 }
@@ -337,7 +343,7 @@ case class DirectorySourcePath(dir: File) extends JFileDirectoryLookup[SourceFil
   def asSourcePathString: String = asClassPathString
 
   protected def createFileEntry(file: AbstractFile): SourceFileEntryImpl = SourceFileEntryImpl(file)
-  protected def isMatchingFile(f: File): Boolean = endsScalaOrJava(f.getName)
+  protected def isMatchingFile(f: File, siblingExists: String => Boolean): Boolean = endsScalaOrJava(f.getName)
 
   override def findClass(className: String): Option[ClassRepresentation] = findSourceFile(className) map SourceFileEntryImpl
 

src/compiler/scala/tools/nsc/classpath/FileUtils.scala

@@ -24,7 +24,7 @@ object FileUtils {
   implicit class AbstractFileOps(val file: AbstractFile) extends AnyVal {
     def isPackage: Boolean = file.isDirectory && mayBeValidPackage(file.name)
 
-    def isClass: Boolean = !file.isDirectory && (file.hasExtension("class") || file.hasExtension("sig"))
+    def isClass: Boolean = !file.isDirectory && (file.hasExtension("class") || file.hasExtension("sig") || file.hasExtension("tasty"))
 
     def isScalaOrJavaSource: Boolean = !file.isDirectory && (file.hasExtension("scala") || file.hasExtension("java"))
 
@@ -46,6 +46,7 @@ object FileUtils {
   private val SUFFIX_SCALA = ".scala"
   private val SUFFIX_JAVA = ".java"
   private val SUFFIX_SIG = ".sig"
+  private val SUFFIX_TASTY = ".tasty"
 
   def stripSourceExtension(fileName: String): String = {
     if (endsScala(fileName)) stripClassExtension(fileName)
@@ -58,7 +59,7 @@ object FileUtils {
   @inline private def ends (filename:String, suffix:String) = filename.endsWith(suffix) && filename.length > suffix.length
 
   def endsClass(fileName: String): Boolean =
-    ends (fileName, SUFFIX_CLASS) || fileName.endsWith(SUFFIX_SIG)
+    ends (fileName, SUFFIX_CLASS) || fileName.endsWith(SUFFIX_SIG) || fileName.endsWith(SUFFIX_TASTY)
 
   def endsScalaOrJava(fileName: String): Boolean =
     endsScala(fileName) || endsJava(fileName)

src/compiler/scala/tools/nsc/classpath/VirtualDirectoryClassPath.scala

@@ -30,6 +30,8 @@ case class VirtualDirectoryClassPath(dir: VirtualDirectory) extends ClassPath wi
     case Some(f) => dir.iterator.filter(f).toArray
     case _ => dir.toArray
   }
+  protected def hasChild(dir: AbstractFile, name: String): Boolean = dir.lookupName(name, directory = false) != null
+
   def getName(f: AbstractFile): String = f.name
   def toAbstractFile(f: AbstractFile): AbstractFile = f
   def isPackage(f: AbstractFile): Boolean = f.isPackage
@@ -47,5 +49,6 @@ case class VirtualDirectoryClassPath(dir: VirtualDirectory) extends ClassPath wi
   private[nsc] def classes(inPackage: PackageName): Seq[ClassFileEntry] = files(inPackage)
 
   protected def createFileEntry(file: AbstractFile): ClassFileEntryImpl = ClassFileEntryImpl(file)
-  protected def isMatchingFile(f: AbstractFile): Boolean = f.isClass
+  protected def isMatchingFile(f: AbstractFile, siblingExists: String => Boolean): Boolean =
+    f.isClass && !(f.hasExtension("class") && siblingExists(f.name.dropRight(6) + ".tasty"))
 }

src/compiler/scala/tools/nsc/classpath/ZipAndJarFileLookupFactory.scala

@@ -73,7 +73,9 @@ object ZipAndJarClassPathFactory extends ZipAndJarFileLookupFactory {
     override private[nsc] def classes(inPackage: PackageName): Seq[ClassFileEntry] = files(inPackage)
 
     override protected def createFileEntry(file: FileZipArchive#Entry): ClassFileEntryImpl = ClassFileEntryImpl(file)
-    override protected def isRequiredFileType(file: AbstractFile): Boolean = file.isClass
+    override protected def isRequiredFileType(file: AbstractFile, siblingExists: String => Boolean): Boolean = {
+      file.isClass && !(file.hasExtension("class") && siblingExists(file.name.dropRight(6) + ".tasty"))
+    }
   }
 
   /**
@@ -182,7 +184,7 @@ object ZipAndJarSourcePathFactory extends ZipAndJarFileLookupFactory {
     override private[nsc] def sources(inPackage: PackageName): Seq[SourceFileEntry] = files(inPackage)
 
     override protected def createFileEntry(file: FileZipArchive#Entry): SourceFileEntryImpl = SourceFileEntryImpl(file)
-    override protected def isRequiredFileType(file: AbstractFile): Boolean = file.isScalaOrJavaSource
+    override protected def isRequiredFileType(file: AbstractFile, siblingExists: String => Boolean): Boolean = file.isScalaOrJavaSource
   }
 
   override protected def createForZipFile(zipFile: AbstractFile, zipSettings: ZipSettings): ClassPath with Closeable = ZipArchiveSourcePath(zipFile.file)

src/compiler/scala/tools/nsc/classpath/ZipArchiveFileLookup.scala

@@ -46,14 +46,14 @@ trait ZipArchiveFileLookup[FileEntryType <: ClassRepresentation] extends Efficie
   protected def files(inPackage: PackageName): Seq[FileEntryType] =
     for {
       dirEntry <- findDirEntry(inPackage).toSeq
-      entry <- dirEntry.iterator if isRequiredFileType(entry)
+      entry <- dirEntry.iterator if isRequiredFileType(entry, dirEntry.entries.contains)
     } yield createFileEntry(entry)
 
   protected def file(inPackage: PackageName, name: String): Option[FileEntryType] =
     findDirEntry(inPackage) match {
       case Some(dirEntry) =>
         val entry = dirEntry.lookupName(name, directory = false)
-        if (entry != null && isRequiredFileType(entry))
+        if (entry != null)
           Some(createFileEntry(entry))
         else
           None
@@ -68,7 +68,7 @@ trait ZipArchiveFileLookup[FileEntryType <: ClassRepresentation] extends Efficie
         for (entry <- dirEntry.iterator) {
           if (entry.isPackage)
             onPackageEntry(PackageEntryImpl(inPackage.entryName(entry.name)))
-          else if (isRequiredFileType(entry))
+          else if (isRequiredFileType(entry, dirEntry.entries.contains))
             onClassesAndSources(createFileEntry(entry))
         }
       case None =>
@@ -81,6 +81,6 @@ trait ZipArchiveFileLookup[FileEntryType <: ClassRepresentation] extends Efficie
 
 
   protected def createFileEntry(file: FileZipArchive#Entry): FileEntryType
-  protected def isRequiredFileType(file: AbstractFile): Boolean
+  protected def isRequiredFileType(file: AbstractFile, siblingExists: String => Boolean): Boolean
 }
 

src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala

@@ -15,22 +15,19 @@ package tools.nsc
 package symtab
 package classfile
 
-import java.io.{ByteArrayOutputStream, IOException}
+import java.io.IOException
 import java.lang.Integer.toHexString
-import java.net.URLClassLoader
-import java.util.UUID
 
 import scala.annotation.switch
 import scala.collection.{immutable, mutable}, mutable.{ArrayBuffer, ListBuffer}
 import scala.reflect.internal.JavaAccFlags
 import scala.reflect.internal.pickling.ByteCodecs
 import scala.reflect.internal.util.ReusableInstance
-import scala.reflect.io.{NoAbstractFile, PlainFile, ZipArchive}
+import scala.reflect.io.NoAbstractFile
 import scala.tools.nsc.Reporting.WarningCategory
 import scala.tools.nsc.io.AbstractFile
 import scala.tools.nsc.util.ClassPath
 import scala.tools.nsc.tasty.{TastyUniverse, TastyUnpickler}
-import scala.tools.tasty.{TastyHeaderUnpickler, TastyReader}
 import scala.util.control.NonFatal
 
 /** This abstract class implements a class file parser.
@@ -73,14 +70,12 @@ abstract class ClassfileParser(reader: ReusableInstance[ReusableDataReader]) {
   protected var staticScope: Scope = _         // the scope of all static definitions
   protected var pool: ConstantPool = _         // the classfile's constant pool
   protected var isScala: Boolean = _           // does class file describe a scala class?
-  protected var isTASTY: Boolean = _           // is this class accompanied by a TASTY file?
   protected var isScalaRaw: Boolean = _        // this class file is a scala class with no pickled info
   protected var busy: Symbol = _               // lock to detect recursive reads
   protected var currentClass: String = _       // JVM name of the current class
   protected var classTParams = Map[Name,Symbol]()
   protected var srcfile0 : Option[AbstractFile] = None
   protected def moduleClass: Symbol = staticModule.moduleClass
-  protected val TASTYUUIDLength: Int = 16
   private var YtastyReader          = false
 
   private def ownerForFlags(jflags: JavaAccFlags) = if (jflags.isStatic) moduleClass else clazz
@@ -163,11 +158,22 @@ abstract class ClassfileParser(reader: ReusableInstance[ReusableDataReader]) {
         this.isScala      = false
         this.YtastyReader = settings.YtastyReader.value
 
-        val magic = in.getInt(in.bp)
-        if (magic != JAVA_MAGIC && file.name.endsWith(".sig")) {
+        val isJavaMagic = in.getInt(in.bp) == JAVA_MAGIC
+        if (!isJavaMagic && file.name.endsWith(".sig")) {
           currentClass = clazz.javaClassName
           isScala = true
           unpickler.unpickle(in.buf.take(file.sizeOption.get), 0, clazz, staticModule, file.name)
+        } else if (!isJavaMagic && file.name.endsWith(".tasty")) {
+          if (!YtastyReader)
+            MissingRequirementError.signal(s"Add -Ytasty-reader to scalac options to parse the TASTy in $file")
+
+          // TODO [tasty]: it seems tests don't fail if we remove this, but previously this
+          // was added for the following reason:
+          // >  Force scala.AnyRef, otherwise we get "error: Symbol AnyRef is missing from the classpath"
+          AnyRefClass
+
+          val bytes = in.buf.take(file.sizeOption.get)
+          TastyUnpickler.unpickle(TastyUniverse)(bytes, clazz, staticModule, file.path.stripSuffix(".class") + ".tasty")
         } else {
           parseHeader()
           this.pool = new ConstantPool
@@ -501,7 +507,7 @@ abstract class ClassfileParser(reader: ReusableInstance[ReusableDataReader]) {
       if (!c.isInstanceOf[StubSymbol] && c != clazz) mismatchError(c)
     }
 
-    if (isScala || isTASTY) {
+    if (isScala) {
       () // We're done
     } else if (isScalaRaw) {
       val decls = clazz.enclosingPackage.info.decls
@@ -1095,8 +1101,6 @@ abstract class ClassfileParser(reader: ReusableInstance[ReusableDataReader]) {
 
     var innersStart = -1
     var runtimeAnnotStart = -1
-    var TASTYAttrStart = -1
-    var TASTYAttrLen = -1
 
     val numAttrs = u2()
     var i = 0
@@ -1110,13 +1114,8 @@ abstract class ClassfileParser(reader: ReusableInstance[ReusableDataReader]) {
         case tpnme.ScalaATTR =>
           isScalaRaw = true
           i = numAttrs
-        case tpnme.TASTYATTR if !YtastyReader =>
-          MissingRequirementError.signal(s"Add -Ytasty-reader to scalac options to parse the TASTy in $file")
         case tpnme.TASTYATTR =>
-          isTASTY = true
-          TASTYAttrLen = attrLen
-          TASTYAttrStart = in.bp
-          i = numAttrs
+          MissingRequirementError.notFound(s"TASTy file for associated class file $file")
         case tpnme.InnerClassesATTR =>
           innersStart = in.bp
         case tpnme.RuntimeAnnotationATTR =>
@@ -1128,13 +1127,6 @@ abstract class ClassfileParser(reader: ReusableInstance[ReusableDataReader]) {
       i += 1
     }
 
-    // To understand the situation, it's helpful to know that:
-    // - Scalac emits the `ScalaSignature` attribute for classfiles with pickled information
-    // and the `Scala` attribute for everything else.
-    // - Dotty emits the `TASTY` attribute for classfiles with pickled information
-    // and the `Scala` attribute for _every_ classfile.
-    isScalaRaw &= !isTASTY
-
     if (isScala) {
       def parseScalaSigBytes(): Array[Byte] = {
         val tag = u1()
@@ -1212,65 +1204,6 @@ abstract class ClassfileParser(reader: ReusableInstance[ReusableDataReader]) {
       AnyRefClass // Force scala.AnyRef, otherwise we get "error: Symbol AnyRef is missing from the classpath"
       assert(bytes != null, s"No Scala(Long)Signature annotation in classfile with ScalaSignature attribute: $clazz")
       unpickler.unpickle(bytes, 0, clazz, staticModule, file.name)
-    } else if (isTASTY) {
-
-      def parseTASTYFile(): Array[Byte] = file.underlyingSource match { // TODO: simplify when #3552 is fixed
-        case None =>
-          reporter.error(NoPosition, "Could not load TASTY from .tasty for virtual file " + file)
-          Array.empty
-        case Some(jar: ZipArchive) => // We are in a jar
-          val cl = new URLClassLoader(Array(jar.toURL), /*parent =*/ null)
-          val path = file.path.stripSuffix(".class") + ".tasty"
-          val stream = cl.getResourceAsStream(path)
-          if (stream != null) {
-            val tastyOutStream = new ByteArrayOutputStream()
-            val buffer = new Array[Byte](1024)
-            var read = stream.read(buffer, 0, buffer.length)
-            while (read != -1) {
-              tastyOutStream.write(buffer, 0, read)
-              read = stream.read(buffer, 0, buffer.length)
-            }
-            tastyOutStream.flush()
-            tastyOutStream.toByteArray
-          } else {
-            reporter.error(NoPosition, s"Could not find $path in $jar")
-            Array.empty
-          }
-        case _ =>
-          val plainFile = new PlainFile(io.File(file.path).changeExtension("tasty"))
-          if (plainFile.exists) plainFile.toByteArray
-          else {
-            reporter.error(NoPosition, "Could not find " + plainFile)
-            Array.empty
-          }
-      }
-
-      def parseTASTYBytes(): Array[Byte] = {
-        assert(TASTYAttrLen == TASTYUUIDLength, "TASTY Attribute is not a UUID")
-        assert(TASTYAttrStart != -1, "no TASTY Annotation position")
-        in.bp = TASTYAttrStart
-        val TASTY = in.nextBytes(TASTYUUIDLength)
-        val TASTYBytes = parseTASTYFile()
-        if (TASTYBytes.isEmpty) {
-          reporter.error(NoPosition, s"No Tasty file found for classfile $file with TASTY Attribute")
-        }
-        val reader = new TastyReader(TASTY, 0, TASTYUUIDLength)
-        val expectedUUID = new UUID(reader.readUncompressedLong(), reader.readUncompressedLong())
-        val tastyUUID = new TastyHeaderUnpickler(TASTYBytes).readHeader()
-        if (expectedUUID != tastyUUID) {
-          loaders.warning(
-            NoPosition,
-            s"$file is out of sync with its TASTy file. Loaded TASTy file. Try cleaning the project to fix this issue",
-            WarningCategory.Other,
-            clazz.fullNameString
-          )
-        }
-        TASTYBytes
-      }
-
-      AnyRefClass // Force scala.AnyRef, otherwise we get "error: Symbol AnyRef is missing from the classpath"
-      val bytes = parseTASTYBytes()
-      TastyUnpickler.unpickle(TastyUniverse)(bytes, clazz, staticModule, file.path.stripSuffix(".class") + ".tasty")
     } else if (!isScalaRaw && innersStart != -1) {
       in.bp = innersStart
       val entries = u2()

src/compiler/scala/tools/nsc/tasty/TastyModes.scala

@@ -36,6 +36,8 @@ object TastyModes {
   final val OpaqueTypeDef: TastyMode = TastyMode(1 << 6)
   /** When reading trees of an annotation */
   final val ReadAnnotationCtor: TastyMode = TastyMode(1 << 7)
+  /** When reading a TASTy file produced from a Java source file (file has JAVAattr attribute) */
+  final val ReadJava: TastyMode = TastyMode(1 << 8)
 
   /** The union of `IndexStats` and `InnerScope` */
   final val IndexScopedStats: TastyMode = IndexStats | InnerScope
@@ -63,6 +65,7 @@ object TastyModes {
         if (mode.is(InnerScope))     sb += "InnerScope"
         if (mode.is(OpaqueTypeDef))  sb += "OpaqueTypeDef"
         if (mode.is(ReadAnnotationCtor)) sb += "ReadAnnotationCtor"
+        if (mode.is(ReadJava)) sb += "ReadJava"
         sb.mkString(" | ")
       }
     }