improvement: fetch missing dependency sources

build.sbt

@@ -487,8 +487,9 @@ lazy val metals = project
       // For reading classpaths.
       // for fetching ch.epfl.scala:bloop-frontend and other library dependencies
       "io.get-coursier" % "interface" % V.coursierInterfaces,
-      // for comparing versions
-      "io.get-coursier" %% "versions" % "0.3.2",
+      // for comparing versions && fetching from sbt maven repository
+      "io.get-coursier" %% "coursier" % V.coursier,
+      "io.get-coursier" %% "coursier-sbt-maven-repository" % V.coursier,
       // for logging
       "com.outr" %% "scribe" % V.scribe,
       "com.outr" %% "scribe-file" % V.scribe,

metals-bench/src/main/scala/bench/ClasspathFuzzBench.scala

@@ -46,7 +46,7 @@ class ClasspathFuzzBench {
   @BenchmarkMode(Array(Mode.SingleShotTime))
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
   def run(): Seq[SymbolInformation] = {
-    symbols.search(query)
+    symbols.search(query, None)
   }
 
 }

metals-bench/src/main/scala/bench/WorkspaceFuzzBench.scala

@@ -37,7 +37,7 @@ class WorkspaceFuzzBench {
   @BenchmarkMode(Array(Mode.SingleShotTime))
   @OutputTimeUnit(TimeUnit.MILLISECONDS)
   def upper(): Seq[SymbolInformation] = {
-    symbols.search(query)
+    symbols.search(query, None)
   }
 
 }

metals/src/main/scala/scala/meta/internal/metals/DefinitionProvider.scala

@@ -175,8 +175,9 @@ final class DefinitionProvider(
         else true
       }
 
+      val dialect = scalaVersionSelector.dialectFromBuildTarget(path)
       val locs = workspaceSearch
-        .searchExactFrom(ident.value, path, token)
+        .searchExactFrom(ident.value, path, token, dialect)
 
       val reducedGuesses =
         if (locs.size > 1)

metals/src/main/scala/scala/meta/internal/metals/ImportedBuild.scala

@@ -81,9 +81,14 @@ object ImportedBuild {
         new JavacOptionsParams(ids)
       )
       sources <- conn.buildTargetSources(new SourcesParams(ids))
-      dependencySources <- conn.buildTargetDependencySources(
+      bspProvidedDependencySources <- conn.buildTargetDependencySources(
         new DependencySourcesParams(ids)
       )
+      dependencySources <- resolveMissingDependencySources(
+        bspProvidedDependencySources,
+        javacOptions,
+        scalacOptions,
+      )
       wrappedSources <- conn.buildTargetWrappedSources(
         new WrappedSourcesParams(ids)
       )
@@ -98,6 +103,39 @@ object ImportedBuild {
       )
     }
 
+  private def resolveMissingDependencySources(
+      dependencySources: DependencySourcesResult,
+      javacOptions: JavacOptionsResult,
+      scalacOptions: ScalacOptionsResult,
+  )(implicit ec: ExecutionContext): Future[DependencySourcesResult] = {
+    val dependencySourcesItems = dependencySources.getItems().asScala.toList
+    val idsLookup = dependencySourcesItems.map(_.getTarget()).toSet
+    val classpaths = javacOptions
+      .getItems()
+      .asScala
+      .map(item => (item.getTarget(), item.getClasspath())) ++
+      scalacOptions
+        .getItems()
+        .asScala
+        .map(item => (item.getTarget(), item.getClasspath()))
+
+    val newItemsFuture =
+      Future.sequence {
+        classpaths.collect {
+          case (id, classpath) if !idsLookup(id) =>
+            for {
+              items <- JarSourcesProvider.fetchSources(
+                classpath.asScala.filter(_.endsWith(".jar")).toSeq
+              )
+            } yield new DependencySourcesItem(id, items.asJava)
+        }
+      }
+
+    newItemsFuture.map { newItems =>
+      new DependencySourcesResult((dependencySourcesItems ++ newItems).asJava)
+    }
+  }
+
   def fromList(data: Seq[ImportedBuild]): ImportedBuild =
     if (data.isEmpty) empty
     else if (data.lengthCompare(1) == 0) data.head

metals/src/main/scala/scala/meta/internal/metals/JarSourcesProvider.scala

@@ -0,0 +1,157 @@
+package scala.meta.internal.metals
+
+import java.net.UnknownHostException
+import java.nio.file.Path
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.TimeoutException
+
+import scala.concurrent.ExecutionContext
+import scala.concurrent.Future
+import scala.util.Success
+import scala.util.Try
+import scala.util.control.NonFatal
+import scala.xml.XML
+
+import scala.meta.internal.metals.MetalsEnrichments._
+import scala.meta.internal.semver.SemVer
+import scala.meta.io.AbsolutePath
+
+import coursier.Fetch
+import coursier.Repositories
+import coursier.cache.ArtifactError
+import coursier.core.Classifier
+import coursier.core.Dependency
+import coursier.core.Module
+import coursier.core.ModuleName
+import coursier.core.Organization
+import coursier.error.CoursierError
+import coursier.maven.SbtMavenRepository
+
+object JarSourcesProvider {
+
+  private val sbtRegex = "sbt-(.*)".r
+
+  def fetchSources(
+      jars: Seq[String]
+  )(implicit ec: ExecutionContext): Future[Seq[String]] = {
+    def sourcesPath(jar: String) = s"${jar.stripSuffix(".jar")}-sources.jar"
+
+    val (haveSources, toDownload) = jars.partition { jar =>
+      sourcesPath(jar).toAbsolutePathSafe.exists(_.exists)
+    }
+
+    val dependencies = toDownload.flatMap { jarPath =>
+      val pomPath = s"${jarPath.stripSuffix(".jar")}.pom"
+      val dependency =
+        for {
+          pom <- pomPath.toAbsolutePathSafe
+          if pom.exists
+          dependency <- getDependency(pom)
+        } yield dependency
+
+      dependency.orElse { jarPath.toAbsolutePathSafe.flatMap(sbtFallback) }
+    }.distinct
+
+    val fetchedSources =
+      Future
+        .sequence {
+          dependencies.map { dep =>
+            fetchDependencySources(dep)
+              .recover { case error: CoursierError =>
+                Option(error.getCause()).map(e => (e, e.getCause())) match {
+                  case Some(
+                        (
+                          _: ArtifactError.DownloadError,
+                          e: UnknownHostException,
+                        )
+                      ) =>
+                    // Fail all if repositories cannot be resolved, e.g. no internet connection.
+                    throw e
+                  case _ =>
+                    scribe.warn(
+                      s"Could not fetch dependency sources for $dep, error: $error"
+                    )
+                }
+                Nil
+              }
+          }
+        }
+        .recover {
+          case _: TimeoutException =>
+            scribe.warn(s"Timeout when fetching dependency sources.")
+            Nil
+          case e: UnknownHostException =>
+            scribe.warn(s"Repository `${e.getMessage()}` is not available.")
+            Nil
+          case NonFatal(e) =>
+            scribe.warn(s"Could not fetch dependency sources, error: $e.")
+            Nil
+        }
+        .map(_.flatten.map(_.toUri().toString()))
+
+    val existingSources = haveSources.map(sourcesPath)
+    fetchedSources.map(_ ++ existingSources)
+
+  }
+
+  private def sbtFallback(jar: AbsolutePath): Option[Dependency] = {
+    val filename = jar.filename.stripSuffix(".jar")
+    filename match {
+      case sbtRegex(versionStr) =>
+        Try(SemVer.Version.fromString(versionStr)) match {
+          // Since `SemVer.Version.fromString` might be able to parse strings, that are not versions,
+          // we check if the result version is a correct parse of the input string.
+          case Success(version) if version.toString == versionStr =>
+            val module = Module(
+              Organization("org.scala-sbt"),
+              ModuleName("sbt"),
+              Map.empty,
+            )
+            Some(Dependency(module, versionStr))
+          case _ => None
+        }
+      case _ => None
+    }
+  }
+
+  private def getDependency(pom: AbsolutePath) = {
+    val xml = XML.loadFile(pom.toFile)
+    val groupId = (xml \ "groupId").text
+    val version = (xml \ "version").text
+    val artifactId = (xml \ "artifactId").text
+    val properties = (xml \ "properties")
+
+    def getProperty(name: String) =
+      properties.map(node => (node \ name).text).find(_.nonEmpty).map(name -> _)
+
+    Option.when(groupId.nonEmpty && version.nonEmpty && artifactId.nonEmpty) {
+      val scalaVersion = getProperty("scalaVersion").toMap
+      val sbtVersion = getProperty("sbtVersion").toMap
+      val attributes = (scalaVersion ++ sbtVersion)
+      Dependency(
+        Module(Organization(groupId), ModuleName(artifactId), attributes),
+        version,
+      )
+    }
+  }
+
+  private val sbtMaven = SbtMavenRepository(Repositories.central)
+  private val metalsPluginSnapshots = SbtMavenRepository(
+    Repositories.sonatype("public")
+  )
+
+  def fetchDependencySources(
+      dependency: Dependency
+  )(implicit ec: ExecutionContext): Future[List[Path]] = {
+    val repositories =
+      List(Repositories.central, sbtMaven, metalsPluginSnapshots)
+    Fetch()
+      .withRepositories(repositories)
+      .withDependencies(Seq(dependency))
+      .addClassifiers(Classifier.sources)
+      .future()
+      .map(_.map(_.toPath()).toList)
+      .withTimeout(1, TimeUnit.MINUTES)
+  }
+
+}

metals/src/main/scala/scala/meta/internal/metals/MetalsLspService.scala

@@ -1677,7 +1677,8 @@ class MetalsLspService(
   ): Future[List[SymbolInformation]] =
     indexingPromise.future.map { _ =>
       val timer = new Timer(time)
-      val result = workspaceSymbols.search(params.getQuery, token).toList
+      val result =
+        workspaceSymbols.search(params.getQuery, token, currentDialect).toList
       if (clientConfig.initialConfig.statistics.isWorkspaceSymbol) {
         scribe.info(
           s"time: found ${result.length} results for query '${params.getQuery}' in $timer"
@@ -1687,9 +1688,12 @@ class MetalsLspService(
     }
 
   def workspaceSymbol(query: String): Seq[SymbolInformation] = {
-    workspaceSymbols.search(query)
+    workspaceSymbols.search(query, currentDialect)
   }
 
+  private def currentDialect =
+    focusedDocument().flatMap(scalaVersionSelector.dialectFromBuildTarget)
+
   def indexSources(): Future[Unit] = Future {
     indexer.indexWorkspaceSources(buildTargets.allWritableData)
   }

metals/src/main/scala/scala/meta/internal/metals/WorkspaceSearchVisitor.scala

@@ -5,6 +5,8 @@ import java.{util => ju}
 
 import scala.collection.mutable
 
+import scala.meta.Dialect
+import scala.meta.dialects
 import scala.meta.internal.metals.MetalsEnrichments._
 import scala.meta.internal.mtags.GlobalSymbolIndex
 import scala.meta.internal.mtags.Symbol
@@ -32,6 +34,7 @@ class WorkspaceSearchVisitor(
     token: CancelChecker,
     index: GlobalSymbolIndex,
     saveClassFileToDisk: Boolean,
+    preferredDialect: Option[Dialect],
 )(implicit rc: ReportContext)
     extends SymbolSearchVisitor {
   private val fromWorkspace = new ju.ArrayList[l.SymbolInformation]()
@@ -88,14 +91,21 @@ class WorkspaceSearchVisitor(
   ): Option[SymbolDefinition] = {
     val nme = Classfile.name(filename)
     val tpe = Symbol(Symbols.Global(pkg, Descriptor.Type(nme)))
-
+    val preferredDialects = preferredDialect match {
+      case Some(dialects.Scala213) =>
+        Set(dialects.Scala213, dialects.Scala213Source3)
+      case Some(dialects.Scala212) =>
+        Set(dialects.Scala212, dialects.Scala212Source3)
+      case opt => opt.toSet
+    }
     val forTpe = index.definitions(tpe)
     val defs = if (forTpe.isEmpty) {
       val term = Symbol(Symbols.Global(pkg, Descriptor.Term(nme)))
       index.definitions(term)
     } else forTpe
-
-    defs.sortBy(_.path.toURI.toString).headOption
+    defs.sortBy { defn =>
+      (!preferredDialects(defn.dialect), defn.path.toURI.toString)
+    }.headOption
   }
   override def shouldVisitPackage(pkg: String): Boolean = true
   override def visitWorkspaceSymbol(

metals/src/main/scala/scala/meta/internal/metals/WorkspaceSymbolProvider.scala

@@ -6,6 +6,7 @@ import java.nio.file.Path
 import scala.collection.concurrent.TrieMap
 import scala.util.control.NonFatal
 
+import scala.meta.Dialect
 import scala.meta.internal.mtags.GlobalSymbolIndex
 import scala.meta.internal.pc.InterruptException
 import scala.meta.io.AbsolutePath
@@ -39,14 +40,21 @@ final class WorkspaceSymbolProvider(
   var inDependencies: ClasspathSearch =
     ClasspathSearch.empty
 
-  def search(query: String): Seq[l.SymbolInformation] = {
-    search(query, () => ())
+  def search(
+      query: String,
+      preferredDialect: Option[Dialect],
+  ): Seq[l.SymbolInformation] = {
+    search(query, () => (), preferredDialect)
   }
 
-  def search(query: String, token: CancelChecker): Seq[l.SymbolInformation] = {
+  def search(
+      query: String,
+      token: CancelChecker,
+      preferredDialect: Option[Dialect],
+  ): Seq[l.SymbolInformation] = {
     if (query.isEmpty) return Nil
     try {
-      searchUnsafe(query, token)
+      searchUnsafe(query, token, preferredDialect)
     } catch {
       case InterruptException() =>
         Nil
@@ -57,6 +65,7 @@ final class WorkspaceSymbolProvider(
       queryString: String,
       path: AbsolutePath,
       token: CancelToken,
+      preferredDialect: Option[Dialect],
   ): Seq[l.SymbolInformation] = {
     val query = WorkspaceSymbolQuery.exact(queryString)
     val visistor =
@@ -66,6 +75,7 @@ final class WorkspaceSymbolProvider(
         token,
         index,
         saveClassFileToDisk,
+        preferredDialect,
       )
     val targetId = buildTargets.inverseSources(path)
     search(query, visistor, targetId)
@@ -205,6 +215,7 @@ final class WorkspaceSymbolProvider(
   private def searchUnsafe(
       textQuery: String,
       token: CancelChecker,
+      preferredDialect: Option[Dialect],
   ): Seq[l.SymbolInformation] = {
     val query = WorkspaceSymbolQuery.fromTextQuery(textQuery)
     val visitor =
@@ -214,6 +225,7 @@ final class WorkspaceSymbolProvider(
         token,
         index,
         saveClassFileToDisk,
+        preferredDialect,
       )
     search(query, visitor, None)
     visitor.allResults()

metals/src/main/scala/scala/meta/internal/metals/scalacli/ScalaCli.scala

@@ -41,7 +41,7 @@ import scala.meta.internal.metals.clients.language.MetalsLanguageClient
 import scala.meta.internal.process.SystemProcess
 import scala.meta.io.AbsolutePath
 
-import coursier.version.Version
+import coursier.core.Version
 
 // todo https://github.com/scalameta/metals/issues/4788
 // clean () =>, use plain values