From 7062b8c9aa9ae46243fcba55ed7a13922784aa58 Mon Sep 17 00:00:00 2001 From: Katarzyna Marek Date: Tue, 7 Nov 2023 18:06:04 +0100 Subject: [PATCH] improvement: fetch missing dependency sources --- .../meta/internal/metals/ImportedBuild.scala | 35 +++++- .../internal/metals/JarSourcesProvider.scala | 106 ++++++++++++++++++ .../test/scala/tests/sbt/SbtServerSuite.scala | 31 ++++- tests/unit/src/main/scala/tests/Library.scala | 23 ++-- .../scala/tests/JarSourcesProviderSuite.scala | 19 ++++ 5 files changed, 202 insertions(+), 12 deletions(-) create mode 100644 metals/src/main/scala/scala/meta/internal/metals/JarSourcesProvider.scala create mode 100644 tests/unit/src/test/scala/tests/JarSourcesProviderSuite.scala diff --git a/metals/src/main/scala/scala/meta/internal/metals/ImportedBuild.scala b/metals/src/main/scala/scala/meta/internal/metals/ImportedBuild.scala index eabee1e5835..a3213bc66f0 100644 --- a/metals/src/main/scala/scala/meta/internal/metals/ImportedBuild.scala +++ b/metals/src/main/scala/scala/meta/internal/metals/ImportedBuild.scala @@ -81,9 +81,14 @@ object ImportedBuild { new JavacOptionsParams(ids) ) sources <- conn.buildTargetSources(new SourcesParams(ids)) - dependencySources <- conn.buildTargetDependencySources( + bspProvidedDependencySources <- conn.buildTargetDependencySources( new DependencySourcesParams(ids) ) + dependencySources <- resolveMissingDependencySources( + bspProvidedDependencySources, + javacOptions, + scalacOptions, + ) wrappedSources <- conn.buildTargetWrappedSources( new WrappedSourcesParams(ids) ) @@ -98,6 +103,34 @@ object ImportedBuild { ) } + private def resolveMissingDependencySources( + dependencySources: DependencySourcesResult, + javacOptions: JavacOptionsResult, + scalacOptions: ScalacOptionsResult, + )(implicit ec: ExecutionContext): Future[DependencySourcesResult] = Future { + val dependencySourcesItems = dependencySources.getItems().asScala.toList + val idsLookup = dependencySourcesItems.map(_.getTarget()).toSet + val classpaths = javacOptions + .getItems() + .asScala + .map(item => (item.getTarget(), item.getClasspath())) ++ + scalacOptions + .getItems() + .asScala + .map(item => (item.getTarget(), item.getClasspath())) + + val newItems = + classpaths.collect { + case (id, classpath) if !idsLookup(id) => + val items = JarSourcesProvider.fetchSources( + classpath.asScala.filter(_.endsWith(".jar")).toSeq + ) + new DependencySourcesItem(id, items.asJava) + } + + new DependencySourcesResult((dependencySourcesItems ++ newItems).asJava) + } + def fromList(data: Seq[ImportedBuild]): ImportedBuild = if (data.isEmpty) empty else if (data.lengthCompare(1) == 0) data.head diff --git a/metals/src/main/scala/scala/meta/internal/metals/JarSourcesProvider.scala b/metals/src/main/scala/scala/meta/internal/metals/JarSourcesProvider.scala new file mode 100644 index 00000000000..d7abafae8b8 --- /dev/null +++ b/metals/src/main/scala/scala/meta/internal/metals/JarSourcesProvider.scala @@ -0,0 +1,106 @@ +package scala.meta.internal.metals + +import java.nio.file.Path + +import scala.util.Success +import scala.util.Try +import scala.xml.XML + +import scala.meta.internal.metals.MetalsEnrichments._ +import scala.meta.internal.semver.SemVer +import scala.meta.io.AbsolutePath + +import coursierapi.Dependency +import coursierapi.Fetch + +object JarSourcesProvider { + + private val sbtRegex = "sbt-(.*)".r + + def fetchSources(jars: Seq[String]): Seq[String] = { + def sourcesPath(jar: String) = s"${jar.stripSuffix(".jar")}-sources.jar" + + val (haveSources, toDownload) = jars.partition { jar => + sourcesPath(jar).toAbsolutePathSafe.exists(_.exists) + } + + val dependencies = toDownload.flatMap { jarPath => + val pomPath = s"${jarPath.stripSuffix(".jar")}.pom" + val dependency = + for { + pom <- pomPath.toAbsolutePathSafe + if pom.exists + dependency <- getDependency(pom) + } yield dependency + + dependency.orElse { jarPath.toAbsolutePathSafe.flatMap(sbtFallback) } + }.distinct + + val fetchedSources = + dependencies.flatMap { dep => + Try(fetchDependencySources(dep)).toEither match { + case Right(fetched) => fetched.map(_.toUri().toString()) + case Left(error) => + scribe.warn( + s"could not fetch dependency sources for $dep, error: $error" + ) + None + } + } + val existingSources = haveSources.map(sourcesPath) + fetchedSources ++ existingSources + + } + + private def sbtFallback(jar: AbsolutePath): Option[Dependency] = { + val filename = jar.filename.stripSuffix(".jar") + filename match { + case sbtRegex(versionStr) if Try().isSuccess => + Try(SemVer.Version.fromString(versionStr)) match { + case Success(version) if version.toString == versionStr => + Some(Dependency.of("org.scala-sbt", "sbt", versionStr)) + case _ => None + } + case _ => None + } + } + + private def getDependency(pom: AbsolutePath) = { + val xml = XML.loadFile(pom.toFile) + val groupId = (xml \ "groupId").text + val version = (xml \ "version").text + val artifactId = (xml \ "artifactId").text + Option + .when(groupId.nonEmpty && version.nonEmpty && artifactId.nonEmpty) { + Dependency.of(groupId, artifactId, version) + } + .filterNot(dep => isSbtDap(dep) || isMetalsPlugin(dep)) + } + + private def isSbtDap(dependency: Dependency) = { + dependency.getModule().getOrganization() == "ch.epfl.scala" && + dependency.getModule().getName() == "sbt-debug-adapter" && + dependency.getVersion() == BuildInfo.debugAdapterVersion + } + + private def isMetalsPlugin(dependency: Dependency) = { + dependency.getModule().getOrganization() == "org.scalameta" && + dependency.getModule().getName() == "sbt-metals" && + dependency.getVersion() == BuildInfo.metalsVersion + } + + private def fetchDependencySources( + dependency: Dependency + ): List[Path] = { + Fetch + .create() + .withDependencies(dependency) + .addClassifiers("sources") + .fetchResult() + .getFiles() + .asScala + .map(_.toPath()) + .toList + } + +} diff --git a/tests/slow/src/test/scala/tests/sbt/SbtServerSuite.scala b/tests/slow/src/test/scala/tests/sbt/SbtServerSuite.scala index 75b47c71884..27c2e4b3e8a 100644 --- a/tests/slow/src/test/scala/tests/sbt/SbtServerSuite.scala +++ b/tests/slow/src/test/scala/tests/sbt/SbtServerSuite.scala @@ -23,13 +23,15 @@ import scribe.writer.Writer import tests.BaseImportSuite import tests.SbtBuildLayout import tests.SbtServerInitializer +import tests.ScriptsAssertions import tests.TestSemanticTokens /** * Basic suite to ensure that a connection to sbt server can be made. */ class SbtServerSuite - extends BaseImportSuite("sbt-server", SbtServerInitializer) { + extends BaseImportSuite("sbt-server", SbtServerInitializer) + with ScriptsAssertions { val preBspVersion = "1.3.13" val supportedMetaBuildVersion = "1.6.0-M1" @@ -497,4 +499,31 @@ class SbtServerSuite } yield () } + test("build-sbt") { + cleanWorkspace() + for { + _ <- initialize( + s"""|/project/build.properties + |sbt.version=${V.sbtVersion} + |/build.sbt + |${SbtBuildLayout.commonSbtSettings} + |ThisBuild / scalaVersion := "${V.scala213}" + |val a = project.in(file("a")) + |/a/src/main/scala/a/A.scala + |package a + |object A { + | val a = 1 + |} + |""".stripMargin + ) + _ <- server.didOpen("build.sbt") + res <- definitionsAt( + "build.sbt", + s"ThisBuild / sc@@alaVersion := \"${V.scala213}\"", + ) + _ = assert(res.length == 1) + _ = assertNoDiff(res.head.getUri().toAbsolutePath.filename, "Keys.scala") + } yield () + } + } diff --git a/tests/unit/src/main/scala/tests/Library.scala b/tests/unit/src/main/scala/tests/Library.scala index 8e66808ee66..4cbd7d416d4 100644 --- a/tests/unit/src/main/scala/tests/Library.scala +++ b/tests/unit/src/main/scala/tests/Library.scala @@ -25,8 +25,11 @@ object Library { Classpath(PackageIndex.bootClasspath.map(AbsolutePath.apply)), Classpath(JdkSources().right.get :: Nil), ) - def catsSources: Seq[AbsolutePath] = - fetchSources("org.typelevel", "cats-core_2.12", "2.0.0-M4") + + def catsDependency: Dependency = + Dependency.of("org.typelevel", "cats-core_2.12", "2.0.0-M4") + def catsSources: Seq[AbsolutePath] = fetchSources(catsDependency) + def cats: Seq[AbsolutePath] = fetch(catsDependency) def scala3: Library = { val binaryVersion = @@ -102,17 +105,17 @@ object Library { ) } - def fetchSources( - org: String, - artifact: String, - version: String, + def fetchSources(dependency: Dependency): Seq[AbsolutePath] = + fetch(dependency, Set("sources")) + + def fetch( + dependency: Dependency, + classifiers: Set[String] = Set.empty, ): Seq[AbsolutePath] = Fetch .create() - .withDependencies( - Dependency.of(org, artifact, version).withTransitive(false) - ) - .withClassifiers(Set("sources").asJava) + .withDependencies(dependency.withTransitive(false)) + .withClassifiers(classifiers.asJava) .fetch() .asScala .toSeq diff --git a/tests/unit/src/test/scala/tests/JarSourcesProviderSuite.scala b/tests/unit/src/test/scala/tests/JarSourcesProviderSuite.scala new file mode 100644 index 00000000000..0fe891455a3 --- /dev/null +++ b/tests/unit/src/test/scala/tests/JarSourcesProviderSuite.scala @@ -0,0 +1,19 @@ +package tests + +import scala.meta.internal.metals.JarSourcesProvider +import scala.meta.internal.metals.MetalsEnrichments._ + +class JarSourcesProviderSuite extends BaseSuite { + + test("download-deps") { + val downloadedSources = + JarSourcesProvider.fetchSources(Library.cats.map(_.toURI.toString())) + assert(downloadedSources.nonEmpty) + downloadedSources.foreach { pathStr => + val path = pathStr.toAbsolutePath + assert(path.exists) + assert(path.filename.endsWith("-sources.jar")) + } + } + +}