diff --git a/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java b/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java index 61c7a1f0..7577982b 100644 --- a/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java +++ b/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java @@ -15,11 +15,14 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.BiFunction; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -79,11 +82,15 @@ public static Path yamlMetadataPath(String version) { return Path.of("_data", "versioned", version.replace('.', '-'), "index", "quarkus.yaml"); } + public static Path yamlVersionMetadataPath() { + return Path.of("_data", "versions.yaml"); + } + public static Path yamlQuarkiverseMetadataPath(String version) { return Path.of("_data", "versioned", version.replace('.', '-'), "index", "quarkiverse.yaml"); } - private final Map allSites; + private final Map allSites; private final Map siteUris; private final CloseableDirectory prefetchedGuides = CloseableDirectory.temp("quarkiverse-guides-"); private final FailureCollector failureCollector; @@ -96,8 +103,11 @@ public QuarkusIO(QuarkusIOConfig config, GitCloneDirectory mainRepository, this.siteUris = Collections.unmodifiableMap(languageUriMap); this.failureCollector = failureCollector; - Map all = new HashMap<>(localizedSites); - all.put(Language.ENGLISH, mainRepository); + Map all = new HashMap<>(); + all.put(Language.ENGLISH, new QuarkusIOCloneDirectory(failureCollector, mainRepository)); + for (var entry : localizedSites.entrySet()) { + all.put(entry.getKey(), new QuarkusIOCloneDirectory(failureCollector, entry.getValue())); + } this.allSites = Collections.unmodifiableMap(all); validateRepositories(mainRepository, localizedSites, failureCollector); @@ -142,9 +152,12 @@ private static void validateRepositories( @Override public void close() throws IOException { + for (QuarkusIOCloneDirectory directory : allSites.values()) { + directory.unprocessed().ifPresent(m -> failureCollector.warning(FailureCollector.Stage.PARSING, m)); + } try (var closer = new Closer()) { closer.push(CloseableDirectory::close, prefetchedGuides); - closer.pushAll(GitCloneDirectory::close, allSites.values()); + closer.pushAll(QuarkusIOCloneDirectory::close, allSites.values()); } } @@ -159,12 +172,14 @@ private Stream versionedGuides() { return allSites.entrySet().stream() .flatMap(entry -> { Language language = entry.getKey(); - GitCloneDirectory cloneDirectory = entry.getValue(); + GitCloneDirectory cloneDirectory = entry.getValue().cloneDirectory(); + VersionFilter versionFilter = entry.getValue().versionFilter(); RevTree translationSourcesTree = cloneDirectory.sourcesTranslationTree(); Repository repository = cloneDirectory.git().getRepository(); return cloneDirectory.sourcesFileStream("_data/versioned", path -> path.endsWith("quarkus.yaml")) .map(QuarkusIO::extractVersion) + .filter(versionFilter) .flatMap(quarkusVersion -> { String quarkus = quarkusVersion.path(); @@ -190,12 +205,14 @@ private Stream legacyGuides() { return allSites.entrySet().stream() .flatMap(entry -> { Language language = entry.getKey(); - GitCloneDirectory cloneDirectory = entry.getValue(); + GitCloneDirectory cloneDirectory = entry.getValue().cloneDirectory(); + VersionFilter versionFilter = entry.getValue().versionFilter(); RevTree translationSourcesTree = cloneDirectory.sourcesTranslationTree(); Repository repository = cloneDirectory.git().getRepository(); return cloneDirectory.sourcesFileStream("_data", path -> path.matches("_data/guides-\\d+-\\d+\\.yaml")) .map(QuarkusIO::extractLegacyVersion) + .filter(versionFilter) .flatMap(quarkusVersion -> { String quarkus = quarkusVersion.path(); @@ -204,7 +221,7 @@ private Stream legacyGuides() { resolveLegacyTranslationPath(quarkusVersion.versionDirectory(), language)); try (InputStream file = cloneDirectory.sourcesFile(quarkus)) { - return parseYamlLegacyMetadata(entry.getValue(), file, quarkusVersion.version(), language, + return parseYamlLegacyMetadata(cloneDirectory, file, quarkusVersion.version(), language, translations); } catch (IOException e) { throw new IllegalStateException( @@ -217,10 +234,13 @@ private Stream legacyGuides() { private Stream quarkiverseGuides() { Language language = Language.ENGLISH; - GitCloneDirectory cloneDirectory = allSites.get(language); + QuarkusIOCloneDirectory quarkusIOCloneDirectory = allSites.get(language); + GitCloneDirectory cloneDirectory = quarkusIOCloneDirectory.cloneDirectory(); + VersionFilter versionFilter = quarkusIOCloneDirectory.versionFilter(); return cloneDirectory.sourcesFileStream("_data/versioned", path -> path.endsWith("quarkiverse.yaml")) .map(QuarkusIO::extractQuarkiverseVersion) + .filter(versionFilter) .flatMap(quarkusVersion -> { String quarkus = quarkusVersion.path(); @@ -239,10 +259,13 @@ private Stream quarkiverseGuides() { private Stream legacyQuarkiverseGuides() { Language language = Language.ENGLISH; - GitCloneDirectory cloneDirectory = allSites.get(language); + QuarkusIOCloneDirectory quarkusIOCloneDirectory = allSites.get(language); + GitCloneDirectory cloneDirectory = quarkusIOCloneDirectory.cloneDirectory(); + VersionFilter versionFilter = quarkusIOCloneDirectory.versionFilter(); return cloneDirectory.sourcesFileStream("_data", path -> path.matches("_data/guides-\\d+-\\d+\\.yaml")) .map(QuarkusIO::extractLegacyVersion) + .filter(versionFilter) .flatMap(quarkusVersion -> { String quarkus = quarkusVersion.path(); @@ -261,10 +284,11 @@ private Stream legacyQuarkiverseGuides() { private Map createTranslations(Function pathCreator) { Map translations = new HashMap<>(); - for (Map.Entry entry : allSites.entrySet()) { + for (Map.Entry entry : allSites.entrySet()) { Language lang = entry.getKey(); + GitCloneDirectory cloneDirectory = entry.getValue().cloneDirectory(); translations.put(lang, translations( - entry.getValue().git().getRepository(), entry.getValue().sourcesTranslationTree(), + cloneDirectory.git().getRepository(), cloneDirectory.sourcesTranslationTree(), pathCreator.apply(lang))); } return translations; @@ -485,6 +509,17 @@ private static Stream parse(InputStream inputStream, return parser.apply(quarkusYaml); } + @SuppressWarnings("unchecked") + private static Set parseVersions(InputStream inputStream) { + Map versionsYaml; + Yaml yaml = new Yaml(); + versionsYaml = yaml.load(inputStream); + + Collection versions = (Collection) versionsYaml.get("documentation"); + + return new LinkedHashSet<>(versions); + } + private Guide createGuide(GitCloneDirectory cloneDirectory, String quarkusVersion, String type, Map parsedGuide, String summaryKey, Language language, Catalog messages) { @@ -582,4 +617,83 @@ private static Set toSet(String value) { record VersionAndPaths(String version, String versionDirectory, String path) { } + record QuarkusIOCloneDirectory(VersionFilter versionFilter, GitCloneDirectory cloneDirectory) implements Closeable { + public QuarkusIOCloneDirectory(FailureCollector failureCollector, GitCloneDirectory cloneDirectory) { + this(VersionFilter.filter(cloneDirectory, failureCollector), cloneDirectory); + } + + @Override + public void close() throws IOException { + try (var closer = new Closer()) { + closer.push(GitCloneDirectory::close, cloneDirectory); + } + } + + public Optional unprocessed() { + Collection unprocessedVersions = versionFilter.unprocessedVersions(); + if (!unprocessedVersions.isEmpty()) { + return Optional.of( + "Not all expected versions were discovered while parsing %s. Missing versions are: %s" + .formatted(cloneDirectory.details(), unprocessedVersions)); + } + return Optional.empty(); + } + } + + private static abstract class VersionFilter implements Predicate { + + private static VersionFilter filter(GitCloneDirectory cloneDirectory, FailureCollector failureCollector) { + try (InputStream inputStream = cloneDirectory.sourcesFile("_data/versions.yaml")) { + Set versions = parseVersions(inputStream); + return new CollectionFilter(versions); + } catch (Exception e) { + failureCollector.warning(FailureCollector.Stage.PARSING, + "Unable to find versions file with explicit list of versions to index within %s, resulting in including all discovered versions." + .formatted(cloneDirectory.toString())); + return AcceptAllFilter.INSTANCE; + + } + } + + public abstract Collection unprocessedVersions(); + + private static class CollectionFilter extends VersionFilter { + + private final Map versions; + + public CollectionFilter(Collection versions) { + this.versions = new HashMap<>(versions.size()); + for (String version : versions) { + this.versions.put(version, false); + } + } + + @Override + public Collection unprocessedVersions() { + return versions.entrySet().stream() + .filter(Predicate.not(Map.Entry::getValue)) + .map(Map.Entry::getKey) + .toList(); + } + + @Override + public boolean test(VersionAndPaths version) { + return Boolean.TRUE.equals(versions.compute(version.version(), (key, value) -> value == null ? null : true)); + } + } + + private static class AcceptAllFilter extends VersionFilter { + public static final VersionFilter INSTANCE = new AcceptAllFilter(); + + @Override + public boolean test(VersionAndPaths s) { + return true; + } + + @Override + public Collection unprocessedVersions() { + return List.of(); + } + } + } } diff --git a/src/test/java/io/quarkus/search/app/SearchServiceTest.java b/src/test/java/io/quarkus/search/app/SearchServiceTest.java index f6f0dc2c..0ccaaec2 100644 --- a/src/test/java/io/quarkus/search/app/SearchServiceTest.java +++ b/src/test/java/io/quarkus/search/app/SearchServiceTest.java @@ -406,7 +406,7 @@ void language() { .extract().body().as(SEARCH_RESULT_SEARCH_HITS); assertThat(result.hits()).extracting(GuideSearchHit::title) .contains("Stork リファレンスガイド", - "Use Hibernate Search with Hibernate ORM and Elasticsearch/OpenSearch"); + "Hibernate ORMとElasticsearch/OpenSearchでHibernate Searchを使用"); } @Test @@ -455,9 +455,9 @@ void searchForPhrase() { @Test void findEnvVariable() { var result = given() - // the variable that we are "planning" to find is actually QUARKUS_DATASOURCE_JDBC_TRACING_IGNORE_FOR_TRACING + // the variable that we are "planning" to find is actually QUARKUS_DATASOURCE_JDBC_URL // But we'll be looking only for a part of it. - .queryParam("q", "QUARKUS_DATASOURCE_JDBC_TRACING_") + .queryParam("q", "QUARKUS_DATASOURCE_JDBC_U") .when().get(GUIDES_SEARCH) .then() .statusCode(200) @@ -470,14 +470,14 @@ void findEnvVariable() { @Test void findConfigProperty() { var result = given() - .queryParam("q", "quarkus.websocket.max-frame-size") + .queryParam("q", "quarkus.vertx.eventbus.tcp-keep-alive") .when().get(GUIDES_SEARCH) .then() .statusCode(200) .extract().body().as(SEARCH_RESULT_SEARCH_HITS); assertThat(result.hits()).extracting(GuideSearchHit::content) .containsOnly( - Set.of("…Default quarkus.websocket.max-frame-size The maximum amount of data that can be sent in a single frame. Messages…")); + Set.of("…false quarkus.vertx.eventbus.tcp-keep-alive Whether to keep the TCP connection opened (keep-alive). Environment…")); } @Test @@ -490,7 +490,7 @@ void findFQCN() { .extract().body().as(SEARCH_RESULT_SEARCH_HITS); assertThat(result.hits()).extracting(GuideSearchHit::content) .containsOnly(Set.of( - "…allow No Javadoc found io.quarkus.deployment.pkg.builditem.NativeImageBuildItem No Javadoc found Path…")); + "…allow No Javadoc found io.quarkus.deployment.pkg.builditem.NativeImageBuildItem No Javadoc found Show…")); } @Test @@ -503,7 +503,7 @@ void findBuildItem() { .extract().body().as(SEARCH_RESULT_SEARCH_HITS); assertThat(result.hits()).extracting(GuideSearchHit::content) .containsOnly(Set.of( - "…allow No Javadoc found io.quarkus.deployment.pkg.builditem.NativeImageBuildItem No Javadoc found Path…")); + "…allow No Javadoc found io.quarkus.deployment.pkg.builditem.NativeImageBuildItem No Javadoc found Show…")); } @Test diff --git a/src/test/java/io/quarkus/search/app/testsupport/QuarkusIOSample.java b/src/test/java/io/quarkus/search/app/testsupport/QuarkusIOSample.java index 37f478da..22d6182c 100644 --- a/src/test/java/io/quarkus/search/app/testsupport/QuarkusIOSample.java +++ b/src/test/java/io/quarkus/search/app/testsupport/QuarkusIOSample.java @@ -14,6 +14,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -291,6 +292,19 @@ private static void yamlQuarkiverseEditor(String version, Path fileToEdit) { }); } + private static void yamlVersionEditor(Path fileToEdit, Collection versions) { + yamlQuarkusEditor(fileToEdit, quarkusYaml -> { + Map filtered = new HashMap<>(); + filtered.put("quarkus", quarkusYaml.get("quarkus")); + filtered.put("graalvm", quarkusYaml.get("graalvm")); + filtered.put("jdk", quarkusYaml.get("jdk")); + filtered.put("maven", quarkusYaml.get("maven")); + + filtered.put("documentation", versions); + return filtered; + }); + } + private static void yamlQuarkusEditor(Path fileToEdit, Function, Map> editor) { Map filtered; try (InputStream inputStream = Files.newInputStream(fileToEdit)) { @@ -384,6 +398,7 @@ protected AbstractGuideRefSetFilterDefinition(String name, GuideRef... guides) { @Override public void define(FilterDefinitionCollector c) { + c.addVersionMetadata(SAMPLED_VERSIONS); for (String version : SAMPLED_VERSIONS) { c.addMetadata(version, guides); if (QuarkusVersions.MAIN.equals(version)) { @@ -437,6 +452,13 @@ public FilterDefinitionCollector addGuide(GuideRef ref, String version) { return this; } + public FilterDefinitionCollector addVersionMetadata(Collection versions) { + String metadataPath = QuarkusIO.yamlVersionMetadataPath().toString(); + addOnSourceBranch(metadataPath, metadataPath); + addMetadataToFilter(metadataPath, path -> yamlVersionEditor(path, versions)); + return this; + } + public FilterDefinitionCollector addMetadata(String version, GuideRef[] guides) { String metadataPath = QuarkusIO.yamlMetadataPath(version).toString(); addOnSourceBranch(metadataPath, metadataPath); diff --git a/src/test/resources/quarkusio-sample-cn.zip b/src/test/resources/quarkusio-sample-cn.zip index d9899023..08738bdd 100644 Binary files a/src/test/resources/quarkusio-sample-cn.zip and b/src/test/resources/quarkusio-sample-cn.zip differ diff --git a/src/test/resources/quarkusio-sample-es.zip b/src/test/resources/quarkusio-sample-es.zip index 1774c315..4056ee36 100644 Binary files a/src/test/resources/quarkusio-sample-es.zip and b/src/test/resources/quarkusio-sample-es.zip differ diff --git a/src/test/resources/quarkusio-sample-ja.zip b/src/test/resources/quarkusio-sample-ja.zip index 1964e693..0725fc67 100644 Binary files a/src/test/resources/quarkusio-sample-ja.zip and b/src/test/resources/quarkusio-sample-ja.zip differ diff --git a/src/test/resources/quarkusio-sample-pt.zip b/src/test/resources/quarkusio-sample-pt.zip index 215eda52..2085a78a 100644 Binary files a/src/test/resources/quarkusio-sample-pt.zip and b/src/test/resources/quarkusio-sample-pt.zip differ diff --git a/src/test/resources/quarkusio-sample.zip b/src/test/resources/quarkusio-sample.zip index 930c486c..7d9ab27a 100644 Binary files a/src/test/resources/quarkusio-sample.zip and b/src/test/resources/quarkusio-sample.zip differ