improvement: fetch missing dependency sources

metals/src/main/scala/scala/meta/internal/metals/ImportedBuild.scala

@@ -81,9 +81,14 @@ object ImportedBuild {
         new JavacOptionsParams(ids)
       )
       sources <- conn.buildTargetSources(new SourcesParams(ids))
-      dependencySources <- conn.buildTargetDependencySources(
+      bspProvidedDependencySources <- conn.buildTargetDependencySources(
         new DependencySourcesParams(ids)
       )
+      dependencySources <- resolveMissingDependencySources(
+        bspProvidedDependencySources,
+        javacOptions,
+        scalacOptions,
+      )
       wrappedSources <- conn.buildTargetWrappedSources(
         new WrappedSourcesParams(ids)
       )
@@ -98,6 +103,34 @@ object ImportedBuild {
       )
     }
 
+  private def resolveMissingDependencySources(
+      dependencySources: DependencySourcesResult,
+      javacOptions: JavacOptionsResult,
+      scalacOptions: ScalacOptionsResult,
+  )(implicit ec: ExecutionContext): Future[DependencySourcesResult] = Future {
+    val dependencySourcesItems = dependencySources.getItems().asScala.toList
+    val idsLookup = dependencySourcesItems.map(_.getTarget()).toSet
+    val classpaths = javacOptions
+      .getItems()
+      .asScala
+      .map(item => (item.getTarget(), item.getClasspath())) ++
+      scalacOptions
+        .getItems()
+        .asScala
+        .map(item => (item.getTarget(), item.getClasspath()))
+
+    val newItems =
+      classpaths.collect {
+        case (id, classpath) if !idsLookup(id) =>
+          val items = JarSourcesProvider.fetchSources(
+            classpath.asScala.filter(_.endsWith(".jar")).toSeq
+          )
+          new DependencySourcesItem(id, items.asJava)
+      }
+
+    new DependencySourcesResult((dependencySourcesItems ++ newItems).asJava)
+  }
+
   def fromList(data: Seq[ImportedBuild]): ImportedBuild =
     if (data.isEmpty) empty
     else if (data.lengthCompare(1) == 0) data.head

metals/src/main/scala/scala/meta/internal/metals/JarSourcesProvider.scala

@@ -0,0 +1,106 @@
+package scala.meta.internal.metals
+
+import java.nio.file.Path
+
+import scala.util.Success
+import scala.util.Try
+import scala.xml.XML
+
+import scala.meta.internal.metals.MetalsEnrichments._
+import scala.meta.internal.semver.SemVer
+import scala.meta.io.AbsolutePath
+
+import coursierapi.Dependency
+import coursierapi.Fetch
+
+object JarSourcesProvider {
+
+  private val sbtRegex = "sbt-(.*)".r
+
+  def fetchSources(jars: Seq[String]): Seq[String] = {
+    def sourcesPath(jar: String) = s"${jar.stripSuffix(".jar")}-sources.jar"
+
+    val (haveSources, toDownload) = jars.partition { jar =>
+      sourcesPath(jar).toAbsolutePathSafe.exists(_.exists)
+    }
+
+    val dependencies = toDownload.flatMap { jarPath =>
+      val pomPath = s"${jarPath.stripSuffix(".jar")}.pom"
+      val dependency =
+        for {
+          pom <- pomPath.toAbsolutePathSafe
+          if pom.exists
+          dependency <- getDependency(pom)
+        } yield dependency
+
+      dependency.orElse { jarPath.toAbsolutePathSafe.flatMap(sbtFallback) }
+    }.distinct
+
+    val fetchedSources =
+      dependencies.flatMap { dep =>
+        Try(fetchDependencySources(dep)).toEither match {
+          case Right(fetched) => fetched.map(_.toUri().toString())
+          case Left(error) =>
+            scribe.warn(
+              s"could not fetch dependency sources for $dep, error: $error"
+            )
+            None
+        }
+      }
+    val existingSources = haveSources.map(sourcesPath)
+    fetchedSources ++ existingSources
+
+  }
+
+  private def sbtFallback(jar: AbsolutePath): Option[Dependency] = {
+    val filename = jar.filename.stripSuffix(".jar")
+    filename match {
+      case sbtRegex(versionStr) if Try().isSuccess =>
+        Try(SemVer.Version.fromString(versionStr)) match {
+          case Success(version) if version.toString == versionStr =>
+            Some(Dependency.of("org.scala-sbt", "sbt", versionStr))
+          case _ => None
+        }
+      case _ => None
+    }
+  }
+
+  private def getDependency(pom: AbsolutePath) = {
+    val xml = XML.loadFile(pom.toFile)
+    val groupId = (xml \ "groupId").text
+    val version = (xml \ "version").text
+    val artifactId = (xml \ "artifactId").text
+    Option
+      .when(groupId.nonEmpty && version.nonEmpty && artifactId.nonEmpty) {
+        Dependency.of(groupId, artifactId, version)
+      }
+      .filterNot(dep => isSbtDap(dep) || isMetalsPlugin(dep))
+  }
+
+  private def isSbtDap(dependency: Dependency) = {
+    dependency.getModule().getOrganization() == "ch.epfl.scala" &&
+    dependency.getModule().getName() == "sbt-debug-adapter" &&
+    dependency.getVersion() == BuildInfo.debugAdapterVersion
+  }
+
+  private def isMetalsPlugin(dependency: Dependency) = {
+    dependency.getModule().getOrganization() == "org.scalameta" &&
+    dependency.getModule().getName() == "sbt-metals" &&
+    dependency.getVersion() == BuildInfo.metalsVersion
+  }
+
+  private def fetchDependencySources(
+      dependency: Dependency
+  ): List[Path] = {
+    Fetch
+      .create()
+      .withDependencies(dependency)
+      .addClassifiers("sources")
+      .fetchResult()
+      .getFiles()
+      .asScala
+      .map(_.toPath())
+      .toList
+  }
+
+}

tests/slow/src/test/scala/tests/sbt/SbtServerSuite.scala

@@ -23,13 +23,15 @@ import scribe.writer.Writer
 import tests.BaseImportSuite
 import tests.SbtBuildLayout
 import tests.SbtServerInitializer
+import tests.ScriptsAssertions
 import tests.TestSemanticTokens
 
 /**
  * Basic suite to ensure that a connection to sbt server can be made.
  */
 class SbtServerSuite
-    extends BaseImportSuite("sbt-server", SbtServerInitializer) {
+    extends BaseImportSuite("sbt-server", SbtServerInitializer)
+    with ScriptsAssertions {
 
   val preBspVersion = "1.3.13"
   val supportedMetaBuildVersion = "1.6.0-M1"
@@ -497,4 +499,31 @@ class SbtServerSuite
     } yield ()
   }
 
+  test("build-sbt") {
+    cleanWorkspace()
+    for {
+      _ <- initialize(
+        s"""|/project/build.properties
+            |sbt.version=${V.sbtVersion}
+            |/build.sbt
+            |${SbtBuildLayout.commonSbtSettings}
+            |ThisBuild / scalaVersion := "${V.scala213}"
+            |val a = project.in(file("a"))
+            |/a/src/main/scala/a/A.scala
+            |package a
+            |object A {
+            | val a = 1
+            |}
+            |""".stripMargin
+      )
+      _ <- server.didOpen("build.sbt")
+      res <- definitionsAt(
+        "build.sbt",
+        s"ThisBuild / sc@@alaVersion := \"${V.scala213}\"",
+      )
+      _ = assert(res.length == 1)
+      _ = assertNoDiff(res.head.getUri().toAbsolutePath.filename, "Keys.scala")
+    } yield ()
+  }
+
 }

tests/unit/src/test/scala/tests/JarSourcesProviderSuite.scala

@@ -0,0 +1,19 @@
+package tests
+
+import scala.meta.internal.metals.JarSourcesProvider
+import scala.meta.internal.metals.MetalsEnrichments._
+
+class JarSourcesProviderSuite extends BaseSuite {
+
+  test("download-deps") {
+    val downloadedSources =
+      JarSourcesProvider.fetchSources(Library.cats.map(_.toURI.toString()))
+    assert(downloadedSources.nonEmpty)
+    downloadedSources.foreach { pathStr =>
+      val path = pathStr.toAbsolutePath
+      assert(path.exists)
+      assert(path.filename.endsWith("-sources.jar"))
+    }
+  }
+
+}