From 5b0cb66d13f3bfdde2971956cbba43d3368f8b91 Mon Sep 17 00:00:00 2001 From: Alexey Loubyansky Date: Thu, 7 Mar 2024 22:39:54 +0100 Subject: [PATCH] Added --redhat-version-rate to the quarkus command --- domino/README.md | 40 ++ .../java/io/quarkus/domino/cli/Quarkus.java | 343 +++++++++++------- 2 files changed, 260 insertions(+), 123 deletions(-) diff --git a/domino/README.md b/domino/README.md index 5cd64ddd..254e29cf 100644 --- a/domino/README.md +++ b/domino/README.md @@ -154,6 +154,46 @@ A custom local Maven repository location can be provided using `--repo-dir` argu java -jar domino.jar quarkus --version=3.2.9.Final --resolve --repo-dir=repo-3.2.9 ``` +#### Red Hat dependency version rate + +`quarkus` command includes option `--redhat-version-rate` that enables calculation of the rate of dependencies with `redhat` version qualifier among all the visited dependencies. + +##### Red Hat build of Quarkus productization rate + +The following command will inspect dependencies of all the Quarkus extensions that are managed by `quarkus-bom`, Quarkus Maven plugin and other Qarkus artifacts that are supported as direct application dependencies (such as `io.quarkus:quarkus-junit5`) and calculate the rate of artifacts containing `redhat` version qualifier: +``` +[user@localhost playground]$ domino quarkus --version 3.2.10.Final-redhat-00002 --members=quarkus-bom --redhat-version-rate + +Total number of dependencies: 1470 +Red Hat version rate: 62.3% +``` + +`--extension-versions=*redhat*` can be added to limit analysis to only extensions that were rebuilt by Red Hat: +``` +[user@localhost playground]$ domino quarkus --version 3.2.10.Final-redhat-00002 --members=quarkus-bom --redhat-version-rate --extension-versions=*redhat* + +Total number of dependencies: 1075 +Red Hat version rate: 82.3% +``` + +##### Red Hat build of Apache Camel for Quarkus productization rate + +The following command will inspect dependencies of all the Camel Quarkus extensions that are managed by `quarkus-camel-bom` and calculate the rate of artifacts containing `redhat` version qualifier: +``` +[user@localhost playground]$ domino quarkus --version 3.2.10.Final-redhat-00002 --members=quarkus-camel-bom --redhat-version-rate + +Total number of dependencies: 2583 +Red Hat version rate: 41.9% +``` + +`--extension-versions=*redhat*` can be added to limit analysis to only extensions that were rebuilt by Red Hat: +``` +[user@localhost playground]$ domino quarkus --version 3.2.10.Final-redhat-00002 --members=quarkus-camel-bom --redhat-version-rate --extension-versions=*redhat* + +Total number of dependencies: 1278 +Red Hat version rate: 80.4% +``` + ### Maven plugin There is also a Maven plugin goal that can be used to generate a dependency report. For example, here is one for Vert.X: diff --git a/domino/app/src/main/java/io/quarkus/domino/cli/Quarkus.java b/domino/app/src/main/java/io/quarkus/domino/cli/Quarkus.java index 60d34c0a..a771756b 100644 --- a/domino/app/src/main/java/io/quarkus/domino/cli/Quarkus.java +++ b/domino/app/src/main/java/io/quarkus/domino/cli/Quarkus.java @@ -31,11 +31,14 @@ import java.util.Properties; import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; import org.eclipse.aether.artifact.Artifact; import org.eclipse.aether.artifact.DefaultArtifact; import org.eclipse.aether.graph.Dependency; import org.eclipse.aether.graph.DependencyNode; +import org.eclipse.aether.repository.RemoteRepository; import picocli.CommandLine; @CommandLine.Command(name = "quarkus", header = "Quarkus platform release analysis", description = "%n" @@ -97,18 +100,17 @@ public class Quarkus implements Callable { "--members" }, description = "Limit the analysis to the specified members", split = ",") protected Set members = Set.of(); + @CommandLine.Option(names = { + "--redhat-version-rate" }, description = "Calculate the rate of redhat versions among the dependencies") + public boolean redhatVersionRate; + protected MessageWriter log = MessageWriter.info(); @Override public Integer call() throws Exception { var resolver = getResolver(); - var platform = QuarkusPlatformInfoReader.builder() - .setResolver(resolver) - .setVersion(version) - .setPlatformKey(platformGroupId) - .build() - .readPlatformInfo(); + var platform = readPlatformInfo(resolver); var memberReports = new ArrayList(members.isEmpty() ? platform.getMembers().size() : members.size()); final MemberReport coreReport = new MemberReport(platform.getCore(), isMemberSelected(platform.getCore())); for (var m : platform.getMembers()) { @@ -117,127 +119,47 @@ public Integer call() throws Exception { } } - final ArtifactSet tracePattern; - if (trace != null && !trace.isEmpty()) { - var builder = ArtifactSet.builder(); - for (var exp : trace) { - builder.include(toArtifactCoordsPattern(exp)); - } - tracePattern = builder.build(); - } else { - tracePattern = null; - } + final ArtifactSet tracePattern = initTracePattern(); final Map> rootsToMembers = tracePattern == null ? Map.of() : new HashMap<>(); - - var treeVisitor = new DependencyTreeVisitor() { - - final Map enforcedBy = new HashMap<>(); - - @Override - public void visit(DependencyTreeVisit visit) { - if (tracePattern != null) { - var result = visit(visit, visit.getRoot()); - if (result != null) { - visit.pushEvent(result); - } - } - } - - private TreeNode visit(DependencyTreeVisit visit, DependencyNode node) { - var a = node.getArtifact(); - TreeNode result = null; - if (tracePattern.contains(a.getGroupId(), a.getArtifactId(), a.getClassifier(), a.getExtension(), - a.getVersion())) { - result = new TreeNode(a, true, getEnforcedInfo(a)); - } - for (var child : node.getChildren()) { - var childResult = visit(visit, child); - if (childResult != null) { - if (result == null) { - result = new TreeNode(a, false); - } - result.addChild(childResult); - } - } - return result; - } - - private String getEnforcedInfo(Artifact a) { - return enforcedBy.computeIfAbsent(a, k -> { - var coords = ArtifactCoords.of(k.getGroupId(), k.getArtifactId(), k.getClassifier(), - k.getExtension(), k.getVersion()); - StringBuilder sb = null; - for (var report : memberReports) { - if ((report.enabled || report == coreReport) && report.bomConstraints.containsKey(coords)) { - if (sb == null) { - sb = new StringBuilder().append(" [managed by "); - } else { - sb.append(", "); - } - sb.append(report.metadata.getBom().getArtifactId()); - } - } - return sb == null ? "" : sb.append("]").toString(); - }); - } - - @Override - public void onEvent(TreeNode root, MessageWriter log) { - if (!rootsToMembers.isEmpty()) { - var a = root.artifact; - var reports = rootsToMembers.get(ArtifactCoords.of(a.getGroupId(), a.getArtifactId(), - a.getClassifier(), a.getExtension(), a.getVersion())); - if (reports != null) { - for (var report : reports) { - report.addTracedExtensionDependency(root); - } - } - } - } - - @Override - public void handleResolutionFailures(Collection requests) { - } - }; - - var treeProcessor = DependencyTreeInspector.configure() - .setArtifactResolver(resolver) - .setResolveDependencies(resolve) - .setParallelProcessing(parallelProcessing) - .setProgressTrackerPrefix("Inspecting ") - .setTreeVisitor(treeVisitor); - + final Map allNodes = redhatVersionRate ? new ConcurrentHashMap<>() : null; + final AtomicInteger redhatVersionsTotal = new AtomicInteger(); + var treeVisitor = initTreeVisitor(tracePattern, allNodes, redhatVersionsTotal, memberReports, coreReport, + rootsToMembers); + var treeInspector = initTreeInspector(resolver, treeVisitor); var coreConstraints = readBomConstraints(platform.getCore().getBom(), resolver); + for (var m : memberReports) { final List effectiveConstraints; if (m.metadata == platform.getCore()) { m.bomConstraints = mapConstraints(coreConstraints); effectiveConstraints = coreConstraints; - var pluginCoords = platform.getMavenPlugin(); - if (isVersionSelected(pluginCoords.getVersion())) { - treeProcessor.inspectPlugin(getAetherArtifact(pluginCoords)); - if (tracePattern != null) { - rootsToMembers.computeIfAbsent(pluginCoords, k -> new ArrayList<>(1)).add(m); - } - } - for (var extraKey : EXTRA_CORE_ARTIFACTS) { - var extraArtifact = ArtifactCoords.of(extraKey.getGroupId(), extraKey.getArtifactId(), - extraKey.getClassifier(), extraKey.getType(), platform.getCore().getQuarkusCoreVersion()); - var d = m.bomConstraints.get(extraArtifact); - if (d == null && RhVersionPattern.isRhVersion(platform.getCore().getQuarkusCoreVersion())) { - extraArtifact = ArtifactCoords.of(extraKey.getGroupId(), extraKey.getArtifactId(), - extraKey.getClassifier(), extraKey.getType(), - RhVersionPattern.ensureNoRhQualifier(platform.getCore().getQuarkusCoreVersion())); - d = m.bomConstraints.get(extraArtifact); - } - if (d == null) { - log.warn("Failed to locate " + extraArtifact + " among " - + platform.getCore().getBom().toCompactCoords() + " constraints"); - } else if (isVersionSelected(d.getArtifact().getVersion())) { + if (m.enabled) { + var pluginCoords = platform.getMavenPlugin(); + if (isVersionSelected(pluginCoords.getVersion())) { + treeInspector.inspectPlugin(getAetherArtifact(pluginCoords)); if (tracePattern != null) { - rootsToMembers.computeIfAbsent(extraArtifact, k -> new ArrayList<>(1)).add(m); + rootsToMembers.computeIfAbsent(pluginCoords, k -> new ArrayList<>(1)).add(m); + } + } + for (var extraKey : EXTRA_CORE_ARTIFACTS) { + var extraArtifact = ArtifactCoords.of(extraKey.getGroupId(), extraKey.getArtifactId(), + extraKey.getClassifier(), extraKey.getType(), platform.getCore().getQuarkusCoreVersion()); + var d = m.bomConstraints.get(extraArtifact); + if (d == null && RhVersionPattern.isRhVersion(platform.getCore().getQuarkusCoreVersion())) { + extraArtifact = ArtifactCoords.of(extraKey.getGroupId(), extraKey.getArtifactId(), + extraKey.getClassifier(), extraKey.getType(), + RhVersionPattern.ensureNoRhQualifier(platform.getCore().getQuarkusCoreVersion())); + d = m.bomConstraints.get(extraArtifact); + } + if (d == null) { + log.warn("Failed to locate " + extraArtifact + " among " + + platform.getCore().getBom().toCompactCoords() + " constraints"); + } else if (isVersionSelected(d.getArtifact().getVersion())) { + if (tracePattern != null) { + rootsToMembers.computeIfAbsent(extraArtifact, k -> new ArrayList<>(1)).add(m); + } + treeInspector.inspectAsDependency(d.getArtifact(), effectiveConstraints, d.getExclusions()); } - treeProcessor.inspectAsDependency(d.getArtifact(), effectiveConstraints, d.getExclusions()); } } } else { @@ -251,7 +173,7 @@ public void handleResolutionFailures(Collection requests) { for (var e : m.metadata.getExtensions()) { if (isVersionSelected(e.getVersion())) { var d = m.bomConstraints.get(e); - treeProcessor.inspectAsDependency(getAetherArtifact(e), effectiveConstraints, + treeInspector.inspectAsDependency(getAetherArtifact(e), effectiveConstraints, d == null ? List.of() : d.getExclusions()); if (tracePattern != null) { rootsToMembers.computeIfAbsent(e, k -> new ArrayList<>(1)).add(m); @@ -282,7 +204,7 @@ public void handleResolutionFailures(Collection requests) { }); } d = m.bomConstraints.get(deployment); - treeProcessor.inspectAsDependency(getAetherArtifact(deployment), effectiveConstraints, + treeInspector.inspectAsDependency(getAetherArtifact(deployment), effectiveConstraints, d == null ? List.of() : d.getExclusions()); if (tracePattern != null) { rootsToMembers.computeIfAbsent(deployment, k -> new ArrayList<>(1)).add(m); @@ -303,8 +225,108 @@ public void handleResolutionFailures(Collection requests) { } } - treeProcessor.complete(); + treeInspector.complete(); + + logMemberReports(memberReports); + + if (allNodes != null) { + log.info("Total number of dependencies: " + allNodes.size()); + if (!allNodes.isEmpty()) { + log.info(String.format("Red Hat version rate: %.1f%%", + ((double) redhatVersionsTotal.get() * 100) / allNodes.size())); + } + log.info(""); + } + + return 0; + } + + private DependencyTreeVisitor initTreeVisitor(ArtifactSet tracePattern, + Map allNodes, AtomicInteger redhatVersionsTotal, + ArrayList memberReports, MemberReport coreReport, + Map> rootsToMembers) { + var treeVisitor = new DependencyTreeVisitor() { + final Map enforcedBy = new HashMap<>(); + + @Override + public void visit(DependencyTreeVisit visit) { + if (tracePattern != null || redhatVersionRate) { + var result = visit(visit, visit.getRoot()); + if (result != null) { + visit.pushEvent(result); + } + } + } + + private TreeNode visit(DependencyTreeVisit visit, DependencyNode node) { + var a = node.getArtifact(); + if (allNodes != null) { + var coords = ArtifactCoords.of(a.getGroupId(), a.getArtifactId(), a.getClassifier(), a.getExtension(), + a.getVersion()); + if (allNodes.put(coords, coords) == null && RhVersionPattern.isRhVersion(a.getVersion())) { + redhatVersionsTotal.incrementAndGet(); + } + } + TreeNode result = null; + if (tracePattern != null + && tracePattern.contains(a.getGroupId(), a.getArtifactId(), a.getClassifier(), a.getExtension(), + a.getVersion())) { + result = new TreeNode(a, true, getEnforcedInfo(a)); + } + for (var child : node.getChildren()) { + var childResult = visit(visit, child); + if (childResult != null) { + if (result == null) { + result = new TreeNode(a, false); + } + result.addChild(childResult); + } + } + return result; + } + + private String getEnforcedInfo(Artifact a) { + return enforcedBy.computeIfAbsent(a, k -> { + var coords = ArtifactCoords.of(k.getGroupId(), k.getArtifactId(), k.getClassifier(), + k.getExtension(), k.getVersion()); + StringBuilder sb = null; + for (var report : memberReports) { + if ((report.enabled || report == coreReport) && report.bomConstraints.containsKey(coords)) { + if (sb == null) { + sb = new StringBuilder().append(" [managed by "); + } else { + sb.append(", "); + } + sb.append(report.metadata.getBom().getArtifactId()); + } + } + return sb == null ? "" : sb.append("]").toString(); + }); + } + + @Override + public void onEvent(TreeNode root, MessageWriter log) { + if (!rootsToMembers.isEmpty()) { + var a = root.artifact; + var reports = rootsToMembers.get(ArtifactCoords.of(a.getGroupId(), a.getArtifactId(), + a.getClassifier(), a.getExtension(), a.getVersion())); + if (reports != null) { + for (var report : reports) { + report.addTracedExtensionDependency(root); + } + } + } + } + + @Override + public void handleResolutionFailures(Collection requests) { + } + }; + return treeVisitor; + } + + private void logMemberReports(ArrayList memberReports) { int membersWithTraces = 0; for (var report : memberReports) { if (report.enabled && report.hasTraces()) { @@ -348,7 +370,39 @@ public void handleResolutionFailures(Collection requests) { } log.info(sb.append(" found").toString()); } - return 0; + } + + private DependencyTreeInspector initTreeInspector(MavenArtifactResolver resolver, + DependencyTreeVisitor treeVisitor) { + return DependencyTreeInspector.configure() + .setArtifactResolver(resolver) + .setResolveDependencies(resolve) + .setParallelProcessing(parallelProcessing) + .setProgressTrackerPrefix("Inspecting ") + .setTreeVisitor(treeVisitor); + } + + private ArtifactSet initTracePattern() { + final ArtifactSet tracePattern; + if (trace != null && !trace.isEmpty()) { + var builder = ArtifactSet.builder(); + for (var exp : trace) { + builder.include(toArtifactCoordsPattern(exp)); + } + tracePattern = builder.build(); + } else { + tracePattern = null; + } + return tracePattern; + } + + private QuarkusPlatformInfo readPlatformInfo(MavenArtifactResolver resolver) { + return QuarkusPlatformInfoReader.builder() + .setResolver(resolver) + .setVersion(version) + .setPlatformKey(platformGroupId) + .build() + .readPlatformInfo(); } private boolean isMemberSelected(QuarkusPlatformInfo.Member member) { @@ -392,6 +446,8 @@ private static List readBomConstraints(ArtifactCoords bom, MavenArti } } + private static final String MRRC_URL = "https://maven.repository.redhat.com/ga"; + private MavenArtifactResolver getResolver() throws BootstrapMavenException { var config = BootstrapMavenContext.config() .setWorkspaceDiscovery(false) @@ -409,7 +465,48 @@ private MavenArtifactResolver getResolver() throws BootstrapMavenException { if (mavenProfiles != null) { System.setProperty(BootstrapMavenOptions.QUARKUS_INTERNAL_MAVEN_CMD_LINE_ARGS, "-P" + mavenProfiles); } - return new MavenArtifactResolver(new BootstrapMavenContext(config)); + var mvnCtx = new BootstrapMavenContext(config); + // if the version is a redhat one, enable the redhat repository in case it's not configured + if (version != null && RhVersionPattern.isRhVersion(version)) { + boolean redhatConfigured = false; + for (var r : mvnCtx.getRemoteRepositories()) { + if (redhatConfigured = isRedhat(r)) { + break; + } + } + if (!redhatConfigured) { + var mrrc = new RemoteRepository.Builder("redhat", "default", MRRC_URL).build(); + mvnCtx = new BootstrapMavenContext( + BootstrapMavenContext.config() + .setRemoteRepositoryManager(mvnCtx.getRemoteRepositoryManager()) + .setRepositorySystem(mvnCtx.getRepositorySystem()) + .setRepositorySystemSession(mvnCtx.getRepositorySystemSession()) + .setRemoteRepositories( + mvnCtx.getRemoteRepositoryManager() + .aggregateRepositories(mvnCtx.getRepositorySystemSession(), + List.of(mrrc), + mvnCtx.getRemoteRepositories(), false)) + .setRemotePluginRepositories( + mvnCtx.getRemoteRepositoryManager() + .aggregateRepositories(mvnCtx.getRepositorySystemSession(), + List.of(mrrc), + mvnCtx.getRemotePluginRepositories(), false))); + } + } + return new MavenArtifactResolver(mvnCtx); + } + + private static boolean isRedhat(RemoteRepository repo) { + // it could be MRRC or another RH repo + if (repo.getUrl().contains("redhat.com")) { + return true; + } + for (var mirrored : repo.getMirroredRepositories()) { + if (isRedhat(mirrored)) { + return true; + } + } + return false; } private static String toCompactCoords(Artifact a) {