SOLR-17602: Per-Module Dependency Locking (#2925)

* Introduce platform module

The platform module includes all dependencies from the
version catalog as constraints and is added to the root
modules as API for transitive inheritance of the constraints.

* Cleanup version catalog

* Fix dependency conflicts when errorprone enabled

* Add comment about platform module

* Enable per-module dependency lockfiles

Removes carrot-search dependencychecks plugin

* Update renovate config to use new task

* Update documentation

* Replace references to versions.props, versions.lock and writeLocks

* Add platform module as dependency to include constraints

* Update lockfiles

.gitattributes

@@ -1,5 +1,6 @@
-# Ignore all differences in line endings for the lock file.
+# Ignore all differences in line endings for lock files.
 versions.lock text eol=lf
+**/*.lockfile text eol=lf
 
 # Gradle files are always in LF.
 *.gradle          text eol=lf

.github/labeler.yml

@@ -2,10 +2,11 @@
 dependencies:
   - changed-files:
       - any-glob-to-any-file:
-        - gradle/libs.versions.toml # Solr 10+
-        - versions.props # Solr < v10
-        - versions.lock
-        - solr/licenses/**
+        - "gradle/libs.versions.toml" # Solr 10+
+        - "**/*.lockfile"
+        - "versions.props" # Solr < v10
+        - "versions.lock"
+        - "solr/licenses/**"
 
 # Add 'documentation' label to any changes within ref-guide or dev-docs
 documentation:

.github/renovate.json

@@ -6,7 +6,7 @@
   "enabledManagers": ["gradle", "github-actions"],
   "includePaths": ["gradle/libs.versions.toml", "versions.*", "build.gradle", ".github/workflows/*"],
   "postUpgradeTasks": {
-    "commands": ["./gradlew writeLocks", "./gradlew updateLicenses"],
+    "commands": ["./gradlew resolveAndLockAll --write-locks", "./gradlew updateLicenses"],
     "fileFilters": ["solr/licenses/*.sha1"],
     "executionMode": "branch"
   },

.github/workflows/bin-solr-test.yml

@@ -38,7 +38,7 @@ jobs:
       with:
         path: |
           ~/.gradle/caches
-        key: ${{ runner.os }}-gradle-binsolr-${{ hashFiles('versions.lock') }}
+        key: ${{ runner.os }}-gradle-binsolr-${{ hashFiles('**/*.lockfile') }}
         restore-keys: |
           ${{ runner.os }}-gradle-binsolr-
           ${{ runner.os }}-gradle-

.github/workflows/docker-test.yml

@@ -42,7 +42,7 @@ jobs:
       with:
         path: |
           ~/.gradle/caches
-        key: ${{ runner.os }}-gradle-docker-${{ hashFiles('versions.lock') }}
+        key: ${{ runner.os }}-gradle-docker-${{ hashFiles('**/*.lockfile') }}
         restore-keys: |
           ${{ runner.os }}-gradle-docker-
           ${{ runner.os }}-gradle-

.github/workflows/gradle-precommit.yml

@@ -36,7 +36,7 @@ jobs:
       with:
         path: |
           ~/.gradle/caches
-        key: ${{ runner.os }}-gradle-precommit-${{ hashFiles('versions.lock') }}
+        key: ${{ runner.os }}-gradle-precommit-${{ hashFiles('**/*.lockfile') }}
         restore-keys: |
           ${{ runner.os }}-gradle-precommit-
           ${{ runner.os }}-gradle-

.github/workflows/solrj-test.yml

@@ -35,7 +35,7 @@ jobs:
       with:
         path: |
           ~/.gradle/caches
-        key: ${{ runner.os }}-gradle-solrj-${{ hashFiles('versions.lock') }}
+        key: ${{ runner.os }}-gradle-solrj-${{ hashFiles('**/*.lockfile') }}
         restore-keys: |
           ${{ runner.os }}-gradle-solrj-
           ${{ runner.os }}-gradle-

build.gradle

@@ -22,7 +22,6 @@ plugins {
   id 'base'
   id 'solr.build-infra'
 
-  alias(libs.plugins.carrotsearch.dependencychecks)
   alias(libs.plugins.owasp.dependencycheck)
   alias(libs.plugins.cutterslade.analyze)
   alias(libs.plugins.benmanes.versions)

dev-docs/dependency-upgrades.adoc

@@ -30,43 +30,40 @@ In order to upgrade a dependency, you need to run through a number of steps:
 
 1. Identify the available versions from e.g. https://search.maven.org[Maven Central]
 2. Update the version in `gradle/libs.versions.toml` file
-3. Run `./gradlew writeLocks` to re-generate `versions.lock`. Note that this may cause a cascading effect where
+3. Run `./gradlew resolveAndLockAll` to re-generate lockfiles. Note that this may cause a cascading effect where
    the locked version of other dependencies also change.
 4. In case of a conflict, resolve the conflict according to `help/dependencies.txt`
-5. Check if there are any constraints that are obsolete after the dependency update
-6. Update the license and notice files of the changed dependencies. See `help/dependencies.txt` for
-   details.
-7. Run `./gradlew updateLicenses` to re-generate SHA1 checksums of the new jar files.
-8. Once in a while, a new version of a dependency will transitively bring in brand-new dependencies.
+5. Update the license and notice files of the changed dependencies. See `help/dependencies.txt` for details.
+6. Run `./gradlew updateLicenses` to re-generate SHA1 checksums of the new jar files.
+7. Once in a while, a new version of a dependency will transitively bring in brand-new dependencies.
    You'll need to decide whether to keep or exclude them. See `help/dependencies.txt` for details.
 
-=== Reviewing Constraints
+=== Constraints and Version Alignment
 
-The constraints are defined in gradle/validation/dependencies.gradle. There, if the updated dependency is listed,
-the constraint can be reviewed, updated or removed.
+To sync the version of direct and transitive dependencies across the project, we iterate in the `:platform` module
+over the libraries defined in `gradle/libs.version.toml` and add them as constraints. Then, we use the module in
+main modules like `:solr:api` and `:solr:core` and transitively pass down to all other modules the constraints.
 
-The constraints fall into two "groups". In the first group there are dependency constraints from dependencies
-that our project directly includes and require version alignment to sync the versions across all transitive
-dependencies. In the second group are dependencies that are only present as transitive dependencies.
-There, we try to follow the convention to provide additional information with "which dependencies use what version",
-so that the next person reviewing the constraint does not have to look it up. However, this is quite time-consuming
-to analyze the dependencies and therefore subject to change.
+If a new module does not depend on another module that already includes `:platform` as a platform dependency, it should
+explicitly add it to sync the versions with the rest of the project. `:solr:server` is one case where this is necessary.
 
-In order to review a constraint, you have to check if the updated dependency is mentioned in any of the constraints,
-either as a reason for another dependency constraint or as the constraint's dependency. Removing temporarily
-a constraint, the task writeLocks will fail if the constraint is still required.
+=== Addressing Security Vulnerabilities
 
-This process and the constraints of dependencies.gradle are not optimal, as it is quite time-consuming and not obvious
-by just looking at it. We just haven't found yet a more efficient way to maintain these constraints.
+When it comes to security vulnerabilities that are found in direct or transitive dependencies, the recommended way to
+address them is to update the specific library if there is a new release that solves this issue. For both direct and
+transitive dependencies, we simply have to update the version as described above.
 
-== Renovate bot Pull Requests
+In case it is a transitive dependency that is not directly used, you can simply add it to `libs.versions.toml` as you
+would with any other dependency. The dependency resolution approach defined in `:platform` will handle the rest.
+Don't forget to add a `# @keep` note with a reference to the vulnerable version and CVE that is fixed with the explicit
+definition of the library and new version. This way it is easier to keep track of unreferenced dependencies in our
+libraries toml file, and we can clean them up once the libraries using the modules are updated.
 
-The renovate bot may be replaced in the future with dependabot and this section may only be relevant for older
-versions (<10.0). See https://lists.apache.org/thread/1sb9ttv3lp57z2yod1htx1fykp5sj73z for updates.
+== Renovate bot Pull Requests
 
 A member of the Solr community operates a Github bot running https://github.com/renovatebot/renovate[Renovate], which
 files Pull Requests to Solr with dependency upgrade proposals. The PRs are labeled `dependencies` and do include
-changes resulting from `./gradlew writeLocks` and `updateLicenses`.
+changes resulting from the gradle tasks `resolveAndLockAll` and `updateLicenses`.
 
 Community members and committers can then review, and if manual changes are needed, help bring the PR to completion.
 For many dependencies, a changelog is included in the PR text, which may help guide the upgrade decision.
@@ -78,9 +75,13 @@ that will get its own separate Pull Request, so you can choose.
 If an upgrade is decided, simply merge (and backport) the PR. To skip an upgrade, close the PR. If a PR is left open,
 it will be re-used and auto updated whenever a newer patch- or minor version gets available. Thus, one can reduce
 churn from frequently-updated dependencies by delaying merge until a few weeks before a new release. One can also
-choose to change to a less frequent schedule or disable the bot, by editing `renovate.json`
+choose to change to a less frequent schedule or disable the bot, by editing `renovate.json`.
+
+Please note that Solr version prior to 10.X use a versions resolution plugin that uses `versions.lock` instead of
+`libs.version.toml`. Therefore, changes cannot be backported via cherry-pick.
 
 === Configuring renovate.json
+
 While the bot runs on a https://github.com/solrbot/renovate-github-action[GitHub repo external to the project],
 the bot behavior can be tailored by editing `.github/renovate.json` in this project.
 See https://docs.renovatebot.com[Renovatebot docs] for available options.

dev-docs/lucene-upgrade.md

@@ -34,10 +34,10 @@ Create a new branch locally e.g. `git checkout -b lucene940 -t origin/main` for
 + org.apache.lucene:*=9.4.0
 ```
 
-### `versions.lock` update
+### lockfiles update
 
 ```
-gradlew :writeLocks
+gradlew :resolveAndLockAll
 ```
 
 ### `solr/licenses` update

dev-tools/scripts/README.md

@@ -111,7 +111,7 @@ of the other tools in this folder.
 
     optional arguments:
       -h, --help         show this help message and exit
-      -l LUCENE_VERSION  Optional lucene version. By default will read versions.props
+      -l LUCENE_VERSION  Optional lucene version. By default will read gradle/libs.versions.toml
 
 ### releasedJirasRegex.py
 

dev-tools/scripts/addVersion.py

@@ -23,7 +23,7 @@
 import argparse
 import re
 from configparser import ConfigParser, ExtendedInterpolation
-from textwrap import dedent
+
 
 def update_changes(filename, new_version, init_changes, headers):
   print('  adding new section to %s...' % filename, end='', flush=True)
@@ -115,7 +115,7 @@ def check_lucene_match_version_tests():
 def read_config(current_version, current_lucene_version):
   parser = argparse.ArgumentParser(description='Add a new version to CHANGES, to Version.java, build.gradle and solrconfig.xml files')
   parser.add_argument('version', type=Version.parse, help='New Solr version')
-  parser.add_argument('-l', dest='lucene_version', type=Version.parse, help='Optional lucene version. By default will read versions.props')
+  parser.add_argument('-l', dest='lucene_version', type=Version.parse, help='Optional lucene version. By default will read gradle/libs.versions.toml')
   newconf = parser.parse_args()
   if not newconf.lucene_version:
     newconf.lucene_version = current_lucene_version

dev-tools/scripts/scriptutil.py

@@ -14,13 +14,16 @@
 # limitations under the License.
 
 import argparse
+import os
 import re
 import subprocess
 import sys
-import os
-from enum import Enum
 import time
-import urllib.request, urllib.error, urllib.parse
+import urllib.error
+import urllib.parse
+import urllib.request
+from enum import Enum
+
 
 class Version(object):
   def __init__(self, major, minor, bugfix, prerelease):
@@ -179,7 +182,7 @@ def attemptDownload(urlString, fileName):
 
 version_prop_re = re.compile(r'baseVersion\s*=\s*([\'"])(.*)\1')
 
-lucene_version_prop_re = re.compile(r'org\.apache\.lucene:\*=(.*?)\n')
+lucene_version_prop_re = re.compile(r'^apache-lucene\s*=\s*"([a-zA-Z0-9\.\-]+)"')
 
 def find_current_version():
   script_path = os.path.dirname(os.path.realpath(__file__))
@@ -190,7 +193,7 @@ def find_current_version():
 def find_current_lucene_version():
   script_path = os.path.dirname(os.path.realpath(__file__))
   top_level_dir = os.path.join(os.path.abspath("%s/" % script_path), os.path.pardir, os.path.pardir)
-  versions_file = open('%s/versions.props' % top_level_dir).read()
+  versions_file = open('%s/gradle/libs.versions.toml' % top_level_dir).read()
   return lucene_version_prop_re.search(versions_file).group(1).strip()
 
 

gradle.lockfile

@@ -0,0 +1,16 @@
+# This is a Gradle generated file for dependency locking.
+# Manual edits can break the build and are not advised.
+# This file is expected to be part of source control.
+commons-cli:commons-cli:1.5.0=ratDeps
+commons-io:commons-io:2.11.0=ratDeps
+junit:junit:3.8.1=javacc
+net.java.dev.javacc:javacc:7.0.12=javacc
+org.apache.commons:commons-collections4:4.4=ratDeps
+org.apache.commons:commons-compress:1.21=ratDeps
+org.apache.commons:commons-lang3:3.12.0=ratDeps
+org.apache.rat:apache-rat-api:0.15=ratDeps
+org.apache.rat:apache-rat-core:0.15=ratDeps
+org.apache.rat:apache-rat-tasks:0.15=ratDeps
+org.apache.rat:apache-rat:0.15=ratDeps
+org.eclipse.jdt:ecj:3.39.0=ecjDeps
+empty=

gradle/documentation/render-javadoc.gradle

@@ -175,6 +175,7 @@ configure(project(":solr:test-framework")) {
 // Disable Javadoc rendering for these projects.
 configure(subprojects.findAll {
   it.path in [
+      ':platform',
       ':solr:solr-ref-guide',
       ':solr:server',
       ':solr:webapp']
@@ -193,6 +194,10 @@ configure(subprojects) {
       task.project.configurations.implementation.allDependencies.withType(ProjectDependency).collect { dep ->
         def otherProject = dep.dependencyProject
         return otherProject.tasks.findByName(task.name)
+      }.findAll {
+        // Do not depend on disabled tasks or tasks that do not exist
+        // :platform module does not have a renderJavaDoc and therefore task is null
+        it?.enabled == true
       }
     }
   }

gradle/libs.versions.toml

@@ -66,15 +66,13 @@ bouncycastle = "1.78.1"
 # @keep Browserify version used in ref-guide
 browserify = "17.0.0"
 carrot2-core = "4.5.1"
-carrotsearch-dependencychecks = "0.0.9"
 carrotsearch-hppc = "0.10.0"
 carrotsearch-randomizedtesting = "2.8.1"
 # @keep for version alignment
 checkerframework = "3.44.0"
 codehaus-woodstox = "4.2.2"
 commons-cli = "1.9.0"
 commons-codec = "1.17.1"
-commons-collections = "3.2.2"
 commons-io = "2.15.1"
 cutterslade-analyze = "1.10.0"
 cybozulabs-langdetect = "1.1-20120112"
@@ -101,8 +99,6 @@ google-cloud-nio = "0.127.3"
 google-cloud-storage = "2.27.0"
 google-errorprone = "2.31.0"
 # @keep for version alignment
-google-failureaccess = "1.0.2"
-# @keep for version alignment
 google-gson = "2.11.0"
 google-guava = "33.1.0-jre"
 # @keep for version alignment
@@ -111,7 +107,6 @@ google-j2objc = "3.0.0"
 google-javaformat = "1.18.1"
 # @keep for version alignment
 google-protobuf = "3.25.3"
-google-re2j = "1.7"
 # @keep Gradle version to run the build
 gradle = "8.10"
 grpc = "1.65.1"
@@ -187,7 +182,6 @@ xerial-snappy = "1.1.10.5"
 
 [plugins]
 benmanes-versions = { id = "com.github.ben-manes.versions", version.ref = "benmanes-versions" }
-carrotsearch-dependencychecks = { id = "com.carrotsearch.gradle.dependencychecks", version.ref = "carrotsearch-dependencychecks" }
 cutterslade-analyze = { id = "ca.cutterslade.analyze", version.ref = "cutterslade-analyze" }
 diffplug-spotless = { id = "com.diffplug.spotless", version.ref = "diffplug-spotless" }
 littlerobots-versioncatalogupdate = { id = "nl.littlerobots.version-catalog-update", version.ref = "littlerobots-versioncatalogupdate" }
@@ -294,7 +288,6 @@ checkerframework-qual = { module = "org.checkerframework:checker-qual", version.
 codehaus-woodstox-stax2api = { module = "org.codehaus.woodstox:stax2-api", version.ref = "codehaus-woodstox" }
 commonscli-commonscli = { module = "commons-cli:commons-cli", version.ref = "commons-cli" }
 commonscodec-commonscodec = { module = "commons-codec:commons-codec", version.ref = "commons-codec" }
-commonscollections-commonscollections = { module = "commons-collections:commons-collections", version.ref = "commons-collections" }
 commonsio-commonsio = { module = "commons-io:commons-io", version.ref = "commons-io" }
 cybozulabs-langdetect = { module = "com.cybozu.labs:langdetect", version.ref = "cybozulabs-langdetect" }
 dropwizard-metrics-core = { module = "io.dropwizard.metrics:metrics-core", version.ref = "dropwizard-metrics" }
@@ -362,7 +355,6 @@ google-j2objc-annotations = { module = "com.google.j2objc:j2objc-annotations", v
 # @keep transitive dependency for version alignment
 google-protobuf-java = { module = "com.google.protobuf:protobuf-java", version.ref = "google-protobuf" }
 google-protobuf-javautils = { module = "com.google.protobuf:protobuf-java-util", version.ref = "google-protobuf" }
-google-re2j = { module = "com.google.re2j:re2j", version.ref = "google-re2j" }
 # @keep transitive dependency for version alignment
 grpc-api = { module = "io.grpc:grpc-api", version.ref = "grpc" }
 # @keep transitive dependency for version alignment
@@ -420,6 +412,7 @@ netty-tcnative-boringssl = { module = "io.netty:netty-tcnative-boringssl-static"
 netty-tcnative-classes = { module = "io.netty:netty-tcnative-classes", version.ref = "netty-tcnative" }
 # @keep transitive dependency for version alignment
 netty-transport-classes-epoll = { module = "io.netty:netty-transport-classes-epoll", version.ref = "netty" }
+# @keep transitive dependency for version alignment
 netty-transport-native-epoll = { module = "io.netty:netty-transport-native-epoll", version.ref = "netty" }
 nimbusds-josejwt = { module = "com.nimbusds:nimbus-jose-jwt", version.ref = "nimbusds-josejwt" }
 openjdk-jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "openjdk-jmh" }