From 6665862384a0142b73144f75d381affe4515556c Mon Sep 17 00:00:00 2001 From: "David M. Lloyd" Date: Thu, 23 Jan 2025 10:13:58 -0600 Subject: [PATCH] WIP --- .../pkg/steps/FormattingXMLStreamWriter.java | 32 ++- .../steps/ModularJarResultBuildSubstep.java | 240 +++++++++++------- 2 files changed, 172 insertions(+), 100 deletions(-) diff --git a/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/FormattingXMLStreamWriter.java b/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/FormattingXMLStreamWriter.java index 8f4ec6c85fe9d..659fde225afbf 100644 --- a/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/FormattingXMLStreamWriter.java +++ b/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/FormattingXMLStreamWriter.java @@ -9,6 +9,7 @@ */ final class FormattingXMLStreamWriter implements XMLStreamWriter { private final XMLStreamWriter delegate; + boolean pendingBreak = false; int level = 0; FormattingXMLStreamWriter(final XMLStreamWriter delegate) { @@ -18,14 +19,14 @@ final class FormattingXMLStreamWriter implements XMLStreamWriter { public void writeStartElement(final String localName) throws XMLStreamException { indent(); delegate.writeStartElement(localName); - delegate.writeCharacters("\n"); + pendingBreak = true; level++; } public void writeStartElement(final String namespaceURI, final String localName) throws XMLStreamException { indent(); delegate.writeStartElement(namespaceURI, localName); - delegate.writeCharacters("\n"); + pendingBreak = true; level++; } @@ -33,27 +34,27 @@ public void writeStartElement(final String prefix, final String localName, final throws XMLStreamException { indent(); delegate.writeStartElement(prefix, namespaceURI, localName); - delegate.writeCharacters("\n"); + pendingBreak = true; level++; } public void writeEmptyElement(final String namespaceURI, final String localName) throws XMLStreamException { indent(); delegate.writeEmptyElement(namespaceURI, localName); - delegate.writeCharacters("\n"); + pendingBreak = true; } public void writeEmptyElement(final String prefix, final String localName, final String namespaceURI) throws XMLStreamException { indent(); delegate.writeEmptyElement(prefix, namespaceURI, localName); - delegate.writeCharacters("\n"); + pendingBreak = true; } public void writeEmptyElement(final String localName) throws XMLStreamException { indent(); delegate.writeEmptyElement(localName); - delegate.writeCharacters("\n"); + pendingBreak = true; } public void writeEndElement() throws XMLStreamException { @@ -64,6 +65,7 @@ public void writeEndElement() throws XMLStreamException { } public void writeEndDocument() throws XMLStreamException { + checkBreak(); delegate.writeEndDocument(); } @@ -104,47 +106,55 @@ public void writeComment(final String data) throws XMLStreamException { } public void writeProcessingInstruction(final String target) throws XMLStreamException { + checkBreak(); delegate.writeProcessingInstruction(target); } public void writeProcessingInstruction(final String target, final String data) throws XMLStreamException { + checkBreak(); delegate.writeProcessingInstruction(target, data); } public void writeCData(final String data) throws XMLStreamException { + checkBreak(); delegate.writeCData(data); } public void writeDTD(final String dtd) throws XMLStreamException { + checkBreak(); delegate.writeDTD(dtd); } public void writeEntityRef(final String name) throws XMLStreamException { + checkBreak(); delegate.writeEntityRef(name); } public void writeStartDocument() throws XMLStreamException { + checkBreak(); delegate.writeStartDocument(); delegate.writeCharacters("\n\n"); } public void writeStartDocument(final String version) throws XMLStreamException { + checkBreak(); delegate.writeStartDocument(version); delegate.writeCharacters("\n\n"); } public void writeStartDocument(final String encoding, final String version) throws XMLStreamException { + checkBreak(); delegate.writeStartDocument(encoding, version); delegate.writeCharacters("\n\n"); } public void writeCharacters(final String text) throws XMLStreamException { + checkBreak(); delegate.writeCharacters(text); } public void writeCharacters(final char[] text, final int start, final int len) throws XMLStreamException { indent(); - delegate.writeCharacters("\n"); delegate.writeCharacters(text, start, len); } @@ -173,8 +183,16 @@ public Object getProperty(final String name) throws IllegalArgumentException { } private void indent() throws XMLStreamException { + checkBreak(); for (int i = 0; i < level; i++) { delegate.writeCharacters(" "); } } + + private void checkBreak() throws XMLStreamException { + if (pendingBreak) { + pendingBreak = false; + delegate.writeCharacters("\n"); + } + } } diff --git a/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/ModularJarResultBuildSubstep.java b/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/ModularJarResultBuildSubstep.java index 283c5df7848ed..a7a9eb4d47619 100644 --- a/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/ModularJarResultBuildSubstep.java +++ b/core/deployment/src/main/java/io/quarkus/deployment/pkg/steps/ModularJarResultBuildSubstep.java @@ -21,12 +21,11 @@ import java.util.Collection; import java.util.Comparator; import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.jar.JarEntry; import java.util.jar.JarOutputStream; import java.util.stream.Collectors; @@ -119,20 +118,14 @@ static JarBuildItem buildModularJar( Map artifactsByModule = new HashMap<>(); ModuleStuff appModule = getModuleStuff(appArtifact).withMainClass(mainClassBuildItem.getClassName()); // todo: temporary hacks until we wire up generated class packages - appModule = appModule.withMorePackages(Set.of( - "io.quarkus.runner", - "io.quarkus.runtime.generated", - "io.quarkus.deployment.steps", - "io.quarkus.rest.runtime", - "io.quarkus.arc.setup", - "io.quarkus.arc.generator", - "io.quarkus.runner.recorded")); - modulesByArtifact.put(appModule.artifactKey(), appModule); - artifactsByModule.put(appModule.name(), appModule); - - List allDependencies = new ArrayList<>(512); - model.getDependenciesWithAnyFlag(DependencyFlags.RUNTIME_CP, DependencyFlags.COMPILE_ONLY) - .forEach(allDependencies::add); + appModule = appModule.withMorePackages(Map.of( + "io.quarkus.runner", Visibility.EXPORTED, + "io.quarkus.runtime.generated", Visibility.EXPORTED, + "io.quarkus.deployment.steps", Visibility.EXPORTED, + "io.quarkus.rest.runtime", Visibility.EXPORTED, + "io.quarkus.arc.setup", Visibility.EXPORTED, + "io.quarkus.arc.generator", Visibility.EXPORTED, + "io.quarkus.runner.recorded", Visibility.EXPORTED)); // impl package -> service name -> impl name Map>> extraServicesByPackage = new HashMap<>(); @@ -162,12 +155,19 @@ static JarBuildItem buildModularJar( } // add services to app module - for (String pkg : appModule.packages()) { + for (String pkg : appModule.packages().keySet()) { if (extraServicesByPackage.containsKey(pkg)) { appModule = appModule.withMoreServices(extraServicesByPackage.get(pkg)); } } + modulesByArtifact.put(appModule.artifactKey(), appModule); + artifactsByModule.put(appModule.name(), appModule); + + List allDependencies = new ArrayList<>(512); + model.getDependenciesWithAnyFlag(DependencyFlags.RUNTIME_CP, DependencyFlags.COMPILE_ONLY) + .forEach(allDependencies::add); + for (ResolvedDependency dependency : allDependencies) { if (!dependency.isRuntimeCp()) { continue; @@ -175,7 +175,7 @@ static JarBuildItem buildModularJar( // index all modules first ModuleStuff depModule = getModuleStuff(dependency); // add extra services - for (String pkg : depModule.packages()) { + for (String pkg : depModule.packages().keySet()) { if (extraServicesByPackage.containsKey(pkg)) { depModule = depModule.withMoreServices(extraServicesByPackage.get(pkg)); } @@ -198,7 +198,7 @@ static JarBuildItem buildModularJar( // todo: TEMPORARY global package index (see https://github.com/quarkusio/quarkus/issues/44657 for info) Map packages = new HashMap<>(); for (ModuleStuff moduleStuff : artifactsByModule.values()) { - moduleStuff.packages().forEach(p -> { + moduleStuff.packages().forEach((p, v) -> { if (packages.containsKey(p)) { log.warnf("Package %s found in both %s and %s", p, moduleStuff.name(), packages.get(p).name()); } else { @@ -247,7 +247,11 @@ static JarBuildItem buildModularJar( moduleDir = buildDir.resolve(Path.of(moduleStuff.name().replace('.', '/'))); } for (Path path : moduleStuff.paths()) { - Path outputPath = moduleDir.resolve(path.getFileName()); + String baseName = moduleStuff.artifactKey().getArtifactId(); + if (baseName.equals("quarkus-bootstrap-runner")) { + baseName = "io.quarkus.bootstrap.runner"; + } + Path outputPath = moduleDir.resolve(baseName + ".jar"); try { Files.createDirectories(outputPath.getParent()); copyRecursive(outputPath, path); @@ -301,7 +305,7 @@ static JarBuildItem buildModularJar( } } // now the module.xml with dep info, if needed - if (moduleStuff.automatic()) { + if (!moduleStuff.bootPath() && (moduleStuff.automatic() || moduleStuff.name().equals("jakarta.ws.rs"))) { try (OutputStream os = Files.newOutputStream(moduleDir.resolve("module.xml"), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { try (OutputStreamWriter osw = new OutputStreamWriter(os, StandardCharsets.UTF_8)) { @@ -310,7 +314,8 @@ static JarBuildItem buildModularJar( xmlOutputFactory.setProperty(XMLOutputFactory.IS_REPAIRING_NAMESPACES, Boolean.TRUE); XMLStreamWriter xml = xmlOutputFactory.createXMLStreamWriter(bw); try (XmlCloser ignored = xml::close) { - writeModuleXml(moduleStuff, modulesByArtifact, new FormattingXMLStreamWriter(xml)); + writeModuleXml(moduleStuff, modulesByArtifact, appModule.name(), + new FormattingXMLStreamWriter(xml)); } } } @@ -335,10 +340,6 @@ private static Map newMap(Object ignored) { return new HashMap<>(); } - private static Set newLinkedHashSet(Object ignored) { - return new LinkedHashSet<>(); - } - private static List newList(Object ignored) { return new ArrayList<>(); } @@ -347,7 +348,10 @@ interface XmlCloser extends AutoCloseable { void close() throws XMLStreamException; } - private static void writeModuleXml(final ModuleStuff moduleStuff, final Map modulesByArtifact, + private static void writeModuleXml( + final ModuleStuff moduleStuff, + final Map modulesByArtifact, + final String appModuleName, final XMLStreamWriter xml) throws XMLStreamException { xml.writeStartDocument(); xml.setDefaultNamespace(NS); @@ -382,16 +386,39 @@ private static void writeModuleXml(final ModuleStuff moduleStuff, final Map { + xml.writeEmptyElement("dependency"); + xml.writeAttribute("name", "io.smallrye.config.core"); + } + case "io.quarkus.arc" -> { + xml.writeEmptyElement("dependency"); + xml.writeAttribute("name", appModuleName); + } + case "io.netty.transport" -> { + xml.writeEmptyElement("dependency"); + xml.writeAttribute("name", "io.quarkus.netty"); + } + case "jakarta.ws.rs" -> { + xml.writeEmptyElement("dependency"); + xml.writeAttribute("name", "io.quarkus.resteasy.reactive.common"); + } + } + xml.writeEndElement(); // dependencies // gather package list - Set packages = moduleStuff.packages(); + Map packages = moduleStuff.packages(); xml.writeStartElement("packages"); - String[] pkgArray = packages.toArray(String[]::new); + String[] pkgArray = packages.keySet().toArray(String[]::new); Arrays.sort(pkgArray); for (String pkg : pkgArray) { - xml.writeEmptyElement("export"); + Visibility vis = packages.get(pkg); + switch (vis) { + case PRIVATE -> xml.writeEmptyElement("private"); + case EXPORTED -> xml.writeEmptyElement("export"); + case OPENED -> xml.writeEmptyElement("open"); + } xml.writeAttribute("package", pkg); - // todo: filter out impl, _private, private_, etc. } xml.writeEndElement(); // packages xml.writeStartElement("uses"); @@ -399,48 +426,24 @@ private static void writeModuleXml(final ModuleStuff moduleStuff, final Map> services = new HashMap<>(); - moduleStuff.content().walkIfContains("META-INF/services", pv -> { - // todo: this is totally wrong - String rp = pv.getRelativePath(); - int ls = rp.lastIndexOf('/'); - if (ls == "META-INF/services".length()) { - String apiName = rp.substring(ls + 1); - try { - LinkedHashSet set = services.computeIfAbsent(apiName, ignored -> new LinkedHashSet<>()); - Files.readAllLines(pv.getPath()).stream().map(s -> { - int hi = s.indexOf('#'); - return hi == -1 ? s.trim() : s.substring(0, hi).trim(); - }).filter(s -> !s.isEmpty()).forEach(set::add); - if (set.isEmpty()) { - services.remove(apiName); - } - } catch (IOException e) { - throw new IllegalStateException("Failed to read services from " + pv.getPath(), e); - } + Map> provided = moduleStuff.provided(); + xml.writeStartElement("provides"); + for (Map.Entry> entry : provided.entrySet()) { + String svc = entry.getKey(); + xml.writeStartElement("provide"); + xml.writeAttribute("name", svc); + for (String impl : entry.getValue()) { + xml.writeEmptyElement("with"); + xml.writeAttribute("name", impl); } - }); - if (!services.isEmpty()) { - xml.writeStartElement("provides"); - for (String key : services.keySet()) { - LinkedHashSet set = services.get(key); - if (!set.isEmpty()) { - xml.writeStartElement("provide"); - xml.writeAttribute("name", key); - for (String impl : set) { - xml.writeEmptyElement("with"); - xml.writeAttribute("name", impl); - } - xml.writeEndElement(); - } - } - xml.writeEndElement(); // provides + xml.writeEndElement(); // provide } + xml.writeEndElement(); // provides xml.writeEndElement(); // module xml.writeEndDocument(); } - private static void indexPackages(final Set packages, final PathTree content) { + private static void indexPackages(final Map packages, final PathTree content) { // todo: this is an absurdly poor way to do this content.walk(vp -> { String rp = vp.getRelativePath(); @@ -455,7 +458,14 @@ private static void indexPackages(final Set packages, final PathTree con if (rp.endsWith(".class")) { int idx = rp.lastIndexOf('/'); if (idx != -1) { - packages.add(rp.substring(0, idx).replace('/', '.')); + String pkgName = rp.substring(0, idx).replace('/', '.'); + final Visibility visibility; + if (pkgName.endsWith(".impl") || pkgName.contains(".impl.")) { + visibility = Visibility.PRIVATE; + } else { + visibility = Visibility.EXPORTED; + } + packages.put(pkgName, visibility); } } }); @@ -483,22 +493,17 @@ private static void copyRecursive(final Path outputPath, final DirectoryStream

{ - if (v == null) { - return null; - } - try (InputStream is = v.getUrl().openStream()) { - return ClassFile.of().parse(is.readAllBytes()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); + ClassModel cm = findModuleInfo(contentTree, "module-info.class"); + if (cm == null) { + // slf4j... + cm = findModuleInfo(contentTree, "META-INF/versions/9/module-info.class"); + } String mainClass = null; String moduleName = null; String classifier = key.getClassifier(); String groupId = key.getGroupId(); String artifactId = key.getArtifactId(); - Set packages; + Map packages; Set used; Map> provides; if (cm != null) { @@ -507,14 +512,23 @@ private static ModuleStuff getModuleStuff(ResolvedDependency dependency) { mainClass = cm.findAttribute(Attributes.moduleMainClass()).map(ModuleMainClassAttribute::mainClass) .map(ClassEntry::name) .map(Utf8Entry::stringValue).orElse(null); + Set exportedPackages = cm.findAttribute(Attributes.module()).map(ModuleAttribute::exports) + .map(l -> l.stream() + .filter(ei -> ei.exportsTo().isEmpty()) + .map(ei -> ei.exportedPackage().name().stringValue().replace('/', '.')) + .collect(Collectors.toSet())) + .orElse(Set.of()); packages = cm.findAttribute(Attributes.modulePackages()).map(a -> a.packages().stream() .map(PackageEntry::name) .map(Utf8Entry::stringValue) .map(p -> p.replace('/', '.')) - .collect(Collectors.toUnmodifiableSet())).orElseGet(() -> { - Set set = new HashSet<>(); - indexPackages(set, contentTree); - return set; + .collect( + Collectors.toUnmodifiableMap(Function.identity(), + p -> exportedPackages.contains(p) ? Visibility.EXPORTED : Visibility.PRIVATE))) + .orElseGet(() -> { + Map map = new HashMap<>(); + indexPackages(map, contentTree); + return map; }); used = cm.findAttribute(Attributes.module()).map(ModuleAttribute::uses).orElse(List.of()).stream() .map(ClassEntry::name) @@ -534,6 +548,24 @@ private static ModuleStuff getModuleStuff(ResolvedDependency dependency) { moduleName = manifestAttributes.automaticModuleName(); mainClass = manifestAttributes.mainClassName(); } + // XXX MANUAL MODULE NAME FIXUPS + if (moduleName == null) { + moduleName = switch (groupId) { + case "io.smallrye.certs" -> switch (artifactId) { + case "smallrye-private-key-pem-parser" -> "io.smallrye.certs.pem.private_keys"; + default -> null; + }; + case "io.smallrye.reactive" -> switch (artifactId) { + case "mutiny-smallrye-context-propagation" -> "io.smallrye.mutiny.context"; + default -> null; + }; + case "org.jboss.logging" -> switch (artifactId) { + case "commons-logging-jboss-logging" -> "org.apache.commons.logging"; + default -> null; + }; + default -> null; + }; + } if (moduleName == null) { // make up a name, I guess // mood: [-_.] @@ -566,7 +598,7 @@ private static ModuleStuff getModuleStuff(ResolvedDependency dependency) { } moduleName = finalNameBuilder.toString(); } - indexPackages(packages = new HashSet<>(), contentTree); + indexPackages(packages = new HashMap<>(), contentTree); provides = contentTree.apply("META-INF/services", pv -> { if (pv == null) { return Map.of(); @@ -587,11 +619,25 @@ private static ModuleStuff getModuleStuff(ResolvedDependency dependency) { used = Set.of(); } // todo: this is a very crappy heuristic; we need a better way to identify bootstrap modules - boolean bootPath = dependency.isClassLoaderParentFirst() && cm != null; + boolean bootPath = dependency.isClassLoaderParentFirst() && cm != null && !moduleName.equals("org.slf4j") + || moduleName.equals("io.quarkus.bootstrap.runner"); return new ModuleStuff(moduleName, cm == null, bootPath, dependency, ArtifactKey.gac(groupId, artifactId, classifier), mainClass, packages, used, provides); } + private static ClassModel findModuleInfo(final PathTree contentTree, final String pathName) { + return contentTree.apply(pathName, v -> { + if (v == null) { + return null; + } + try (InputStream is = v.getUrl().openStream()) { + return ClassFile.of().parse(is.readAllBytes()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + } + private static > C readServicesFile(InputStream stream, C output) throws IOException { return readServicesFile(new InputStreamReader(stream, StandardCharsets.UTF_8), output); } @@ -633,11 +679,11 @@ private static > C readServicesFile(BufferedReader * @param provided the provided services */ record ModuleStuff(String name, boolean automatic, boolean bootPath, ResolvedDependency dependency, - ArtifactKey artifactKey, String mainClass, Set packages, Set used, + ArtifactKey artifactKey, String mainClass, Map packages, Set used, Map> provided) { ModuleStuff { - packages = Set.copyOf(packages); + packages = Map.copyOf(packages); used = Set.copyOf(used); provided = Map.copyOf(provided); } @@ -658,15 +704,17 @@ PathTree content() { return dependency.getContentTree(); } - ModuleStuff withMorePackages(final Set morePackages) { - Set packages = Stream.concat( - this.packages.stream(), - morePackages.stream()).collect(Collectors.toUnmodifiableSet()); + ModuleStuff withMorePackages(final Map morePackages) { + var packages = Stream.concat( + this.packages.entrySet().stream(), + morePackages.entrySet().stream()).collect( + Collectors.toUnmodifiableMap( + Map.Entry::getKey, Map.Entry::getValue)); return new ModuleStuff(name, automatic, bootPath, dependency, artifactKey, mainClass, packages, used, provided); } ModuleStuff withMoreServices(final Map> moreServices) { - Map> merged = Stream.concat( + var merged = Stream.concat( provided.entrySet().stream(), moreServices.entrySet().stream()).collect( Collectors.toUnmodifiableMap( @@ -676,4 +724,10 @@ ModuleStuff withMoreServices(final Map> moreServices) { return new ModuleStuff(name, automatic, bootPath, dependency, artifactKey, mainClass, packages, used, merged); } } + + enum Visibility { + PRIVATE, + EXPORTED, + OPENED, + } }