Merge pull request #352 from marko-bekhta/feat/use-versions-to-filter

Add a version filter while parsing guides

src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java

@@ -15,11 +15,14 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.function.BiFunction;
 import java.util.function.Function;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -79,11 +82,15 @@ public static Path yamlMetadataPath(String version) {
         return Path.of("_data", "versioned", version.replace('.', '-'), "index", "quarkus.yaml");
     }
 
+    public static Path yamlVersionMetadataPath() {
+        return Path.of("_data", "versions.yaml");
+    }
+
     public static Path yamlQuarkiverseMetadataPath(String version) {
         return Path.of("_data", "versioned", version.replace('.', '-'), "index", "quarkiverse.yaml");
     }
 
-    private final Map<Language, GitCloneDirectory> allSites;
+    private final Map<Language, QuarkusIOCloneDirectory> allSites;
     private final Map<Language, URI> siteUris;
     private final CloseableDirectory prefetchedGuides = CloseableDirectory.temp("quarkiverse-guides-");
     private final FailureCollector failureCollector;
@@ -96,8 +103,11 @@ public QuarkusIO(QuarkusIOConfig config, GitCloneDirectory mainRepository,
         this.siteUris = Collections.unmodifiableMap(languageUriMap);
         this.failureCollector = failureCollector;
 
-        Map<Language, GitCloneDirectory> all = new HashMap<>(localizedSites);
-        all.put(Language.ENGLISH, mainRepository);
+        Map<Language, QuarkusIOCloneDirectory> all = new HashMap<>();
+        all.put(Language.ENGLISH, new QuarkusIOCloneDirectory(failureCollector, mainRepository));
+        for (var entry : localizedSites.entrySet()) {
+            all.put(entry.getKey(), new QuarkusIOCloneDirectory(failureCollector, entry.getValue()));
+        }
         this.allSites = Collections.unmodifiableMap(all);
 
         validateRepositories(mainRepository, localizedSites, failureCollector);
@@ -142,9 +152,12 @@ private static void validateRepositories(
 
     @Override
     public void close() throws IOException {
+        for (QuarkusIOCloneDirectory directory : allSites.values()) {
+            directory.unprocessed().ifPresent(m -> failureCollector.warning(FailureCollector.Stage.PARSING, m));
+        }
         try (var closer = new Closer<IOException>()) {
             closer.push(CloseableDirectory::close, prefetchedGuides);
-            closer.pushAll(GitCloneDirectory::close, allSites.values());
+            closer.pushAll(QuarkusIOCloneDirectory::close, allSites.values());
         }
     }
 
@@ -159,12 +172,14 @@ private Stream<Guide> versionedGuides() {
         return allSites.entrySet().stream()
                 .flatMap(entry -> {
                     Language language = entry.getKey();
-                    GitCloneDirectory cloneDirectory = entry.getValue();
+                    GitCloneDirectory cloneDirectory = entry.getValue().cloneDirectory();
+                    VersionFilter versionFilter = entry.getValue().versionFilter();
                     RevTree translationSourcesTree = cloneDirectory.sourcesTranslationTree();
                     Repository repository = cloneDirectory.git().getRepository();
 
                     return cloneDirectory.sourcesFileStream("_data/versioned", path -> path.endsWith("quarkus.yaml"))
                             .map(QuarkusIO::extractVersion)
+                            .filter(versionFilter)
                             .flatMap(quarkusVersion -> {
                                 String quarkus = quarkusVersion.path();
 
@@ -190,12 +205,14 @@ private Stream<Guide> legacyGuides() {
         return allSites.entrySet().stream()
                 .flatMap(entry -> {
                     Language language = entry.getKey();
-                    GitCloneDirectory cloneDirectory = entry.getValue();
+                    GitCloneDirectory cloneDirectory = entry.getValue().cloneDirectory();
+                    VersionFilter versionFilter = entry.getValue().versionFilter();
                     RevTree translationSourcesTree = cloneDirectory.sourcesTranslationTree();
                     Repository repository = cloneDirectory.git().getRepository();
 
                     return cloneDirectory.sourcesFileStream("_data", path -> path.matches("_data/guides-\\d+-\\d+\\.yaml"))
                             .map(QuarkusIO::extractLegacyVersion)
+                            .filter(versionFilter)
                             .flatMap(quarkusVersion -> {
                                 String quarkus = quarkusVersion.path();
 
@@ -204,7 +221,7 @@ private Stream<Guide> legacyGuides() {
                                         resolveLegacyTranslationPath(quarkusVersion.versionDirectory(), language));
 
                                 try (InputStream file = cloneDirectory.sourcesFile(quarkus)) {
-                                    return parseYamlLegacyMetadata(entry.getValue(), file, quarkusVersion.version(), language,
+                                    return parseYamlLegacyMetadata(cloneDirectory, file, quarkusVersion.version(), language,
                                             translations);
                                 } catch (IOException e) {
                                     throw new IllegalStateException(
@@ -217,10 +234,13 @@ private Stream<Guide> legacyGuides() {
 
     private Stream<Guide> quarkiverseGuides() {
         Language language = Language.ENGLISH;
-        GitCloneDirectory cloneDirectory = allSites.get(language);
+        QuarkusIOCloneDirectory quarkusIOCloneDirectory = allSites.get(language);
+        GitCloneDirectory cloneDirectory = quarkusIOCloneDirectory.cloneDirectory();
+        VersionFilter versionFilter = quarkusIOCloneDirectory.versionFilter();
 
         return cloneDirectory.sourcesFileStream("_data/versioned", path -> path.endsWith("quarkiverse.yaml"))
                 .map(QuarkusIO::extractQuarkiverseVersion)
+                .filter(versionFilter)
                 .flatMap(quarkusVersion -> {
                     String quarkus = quarkusVersion.path();
 
@@ -239,10 +259,13 @@ private Stream<Guide> quarkiverseGuides() {
 
     private Stream<Guide> legacyQuarkiverseGuides() {
         Language language = Language.ENGLISH;
-        GitCloneDirectory cloneDirectory = allSites.get(language);
+        QuarkusIOCloneDirectory quarkusIOCloneDirectory = allSites.get(language);
+        GitCloneDirectory cloneDirectory = quarkusIOCloneDirectory.cloneDirectory();
+        VersionFilter versionFilter = quarkusIOCloneDirectory.versionFilter();
 
         return cloneDirectory.sourcesFileStream("_data", path -> path.matches("_data/guides-\\d+-\\d+\\.yaml"))
                 .map(QuarkusIO::extractLegacyVersion)
+                .filter(versionFilter)
                 .flatMap(quarkusVersion -> {
                     String quarkus = quarkusVersion.path();
 
@@ -261,10 +284,11 @@ private Stream<Guide> legacyQuarkiverseGuides() {
 
     private Map<Language, Catalog> createTranslations(Function<Language, String> pathCreator) {
         Map<Language, Catalog> translations = new HashMap<>();
-        for (Map.Entry<Language, GitCloneDirectory> entry : allSites.entrySet()) {
+        for (Map.Entry<Language, QuarkusIOCloneDirectory> entry : allSites.entrySet()) {
             Language lang = entry.getKey();
+            GitCloneDirectory cloneDirectory = entry.getValue().cloneDirectory();
             translations.put(lang, translations(
-                    entry.getValue().git().getRepository(), entry.getValue().sourcesTranslationTree(),
+                    cloneDirectory.git().getRepository(), cloneDirectory.sourcesTranslationTree(),
                     pathCreator.apply(lang)));
         }
         return translations;
@@ -485,6 +509,17 @@ private static Stream<Guide> parse(InputStream inputStream,
         return parser.apply(quarkusYaml);
     }
 
+    @SuppressWarnings("unchecked")
+    private static Set<String> parseVersions(InputStream inputStream) {
+        Map<String, Object> versionsYaml;
+        Yaml yaml = new Yaml();
+        versionsYaml = yaml.load(inputStream);
+
+        Collection<String> versions = (Collection<String>) versionsYaml.get("documentation");
+
+        return new LinkedHashSet<>(versions);
+    }
+
     private Guide createGuide(GitCloneDirectory cloneDirectory, String quarkusVersion, String type,
             Map<String, Object> parsedGuide,
             String summaryKey, Language language, Catalog messages) {
@@ -582,4 +617,83 @@ private static Set<String> toSet(String value) {
     record VersionAndPaths(String version, String versionDirectory, String path) {
     }
 
+    record QuarkusIOCloneDirectory(VersionFilter versionFilter, GitCloneDirectory cloneDirectory) implements Closeable {
+        public QuarkusIOCloneDirectory(FailureCollector failureCollector, GitCloneDirectory cloneDirectory) {
+            this(VersionFilter.filter(cloneDirectory, failureCollector), cloneDirectory);
+        }
+
+        @Override
+        public void close() throws IOException {
+            try (var closer = new Closer<IOException>()) {
+                closer.push(GitCloneDirectory::close, cloneDirectory);
+            }
+        }
+
+        public Optional<String> unprocessed() {
+            Collection<String> unprocessedVersions = versionFilter.unprocessedVersions();
+            if (!unprocessedVersions.isEmpty()) {
+                return Optional.of(
+                        "Not all expected versions were discovered while parsing %s. Missing versions are: %s"
+                                .formatted(cloneDirectory.details(), unprocessedVersions));
+            }
+            return Optional.empty();
+        }
+    }
+
+    private static abstract class VersionFilter implements Predicate<VersionAndPaths> {
+
+        private static VersionFilter filter(GitCloneDirectory cloneDirectory, FailureCollector failureCollector) {
+            try (InputStream inputStream = cloneDirectory.sourcesFile("_data/versions.yaml")) {
+                Set<String> versions = parseVersions(inputStream);
+                return new CollectionFilter(versions);
+            } catch (Exception e) {
+                failureCollector.warning(FailureCollector.Stage.PARSING,
+                        "Unable to find versions file with explicit list of versions to index within %s, resulting in including all discovered versions."
+                                .formatted(cloneDirectory.toString()));
+                return AcceptAllFilter.INSTANCE;
+
+            }
+        }
+
+        public abstract Collection<String> unprocessedVersions();
+
+        private static class CollectionFilter extends VersionFilter {
+
+            private final Map<String, Boolean> versions;
+
+            public CollectionFilter(Collection<String> versions) {
+                this.versions = new HashMap<>(versions.size());
+                for (String version : versions) {
+                    this.versions.put(version, false);
+                }
+            }
+
+            @Override
+            public Collection<String> unprocessedVersions() {
+                return versions.entrySet().stream()
+                        .filter(Predicate.not(Map.Entry::getValue))
+                        .map(Map.Entry::getKey)
+                        .toList();
+            }
+
+            @Override
+            public boolean test(VersionAndPaths version) {
+                return Boolean.TRUE.equals(versions.compute(version.version(), (key, value) -> value == null ? null : true));
+            }
+        }
+
+        private static class AcceptAllFilter extends VersionFilter {
+            public static final VersionFilter INSTANCE = new AcceptAllFilter();
+
+            @Override
+            public boolean test(VersionAndPaths s) {
+                return true;
+            }
+
+            @Override
+            public Collection<String> unprocessedVersions() {
+                return List.of();
+            }
+        }
+    }
 }

src/test/java/io/quarkus/search/app/SearchServiceTest.java

@@ -406,7 +406,7 @@ void language() {
                 .extract().body().as(SEARCH_RESULT_SEARCH_HITS);
         assertThat(result.hits()).extracting(GuideSearchHit::title)
                 .contains("Stork リファレンス<span class=\"highlighted\">ガイド</span>",
-                        "Use Hibernate Search with Hibernate ORM and Elasticsearch/OpenSearch");
+                        "Hibernate ORMとElasticsearch/OpenSearchでHibernate Searchを使用");
     }
 
     @Test
@@ -455,9 +455,9 @@ void searchForPhrase() {
     @Test
     void findEnvVariable() {
         var result = given()
-                // the variable that we are "planning" to find is actually QUARKUS_DATASOURCE_JDBC_TRACING_IGNORE_FOR_TRACING
+                // the variable that we are "planning" to find is actually QUARKUS_DATASOURCE_JDBC_URL
                 // But we'll be looking only for a part of it.
-                .queryParam("q", "QUARKUS_DATASOURCE_JDBC_TRACING_")
+                .queryParam("q", "QUARKUS_DATASOURCE_JDBC_U")
                 .when().get(GUIDES_SEARCH)
                 .then()
                 .statusCode(200)
@@ -470,14 +470,14 @@ void findEnvVariable() {
     @Test
     void findConfigProperty() {
         var result = given()
-                .queryParam("q", "quarkus.websocket.max-frame-size")
+                .queryParam("q", "quarkus.vertx.eventbus.tcp-keep-alive")
                 .when().get(GUIDES_SEARCH)
                 .then()
                 .statusCode(200)
                 .extract().body().as(SEARCH_RESULT_SEARCH_HITS);
         assertThat(result.hits()).extracting(GuideSearchHit::content)
                 .containsOnly(
-                        Set.of("…Default <span class=\"highlighted\">quarkus.websocket.max</span>-<span class=\"highlighted\">frame</span>-<span class=\"highlighted\">size</span> The maximum amount of data that can be sent in a single <span class=\"highlighted\">frame</span>. Messages…"));
+                        Set.of("…false <span class=\"highlighted\">quarkus.vertx.eventbus.tcp</span>-<span class=\"highlighted\">keep</span>-<span class=\"highlighted\">alive</span> Whether to <span class=\"highlighted\">keep</span> the TCP connection opened (<span class=\"highlighted\">keep</span>-<span class=\"highlighted\">alive</span>). Environment…"));
     }
 
     @Test
@@ -490,7 +490,7 @@ void findFQCN() {
                 .extract().body().as(SEARCH_RESULT_SEARCH_HITS);
         assertThat(result.hits()).extracting(GuideSearchHit::content)
                 .containsOnly(Set.of(
-                        "…allow No Javadoc found <span class=\"highlighted\">io.quarkus.deployment.pkg.builditem.NativeImageBuildItem</span> No Javadoc found Path…"));
+                        "…allow No Javadoc found <span class=\"highlighted\">io.quarkus.deployment.pkg.builditem.NativeImageBuildItem</span> No Javadoc found Show…"));
     }
 
     @Test
@@ -503,7 +503,7 @@ void findBuildItem() {
                 .extract().body().as(SEARCH_RESULT_SEARCH_HITS);
         assertThat(result.hits()).extracting(GuideSearchHit::content)
                 .containsOnly(Set.of(
-                        "…allow No Javadoc found <span class=\"highlighted\">io.quarkus.deployment.pkg.builditem.NativeImageBuildItem</span> No Javadoc found Path…"));
+                        "…allow No Javadoc found <span class=\"highlighted\">io.quarkus.deployment.pkg.builditem.NativeImageBuildItem</span> No Javadoc found Show…"));
     }
 
     @Test

src/test/java/io/quarkus/search/app/testsupport/QuarkusIOSample.java

@@ -14,6 +14,7 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
@@ -291,6 +292,19 @@ private static void yamlQuarkiverseEditor(String version, Path fileToEdit) {
         });
     }
 
+    private static void yamlVersionEditor(Path fileToEdit, Collection<String> versions) {
+        yamlQuarkusEditor(fileToEdit, quarkusYaml -> {
+            Map<String, Object> filtered = new HashMap<>();
+            filtered.put("quarkus", quarkusYaml.get("quarkus"));
+            filtered.put("graalvm", quarkusYaml.get("graalvm"));
+            filtered.put("jdk", quarkusYaml.get("jdk"));
+            filtered.put("maven", quarkusYaml.get("maven"));
+
+            filtered.put("documentation", versions);
+            return filtered;
+        });
+    }
+
     private static void yamlQuarkusEditor(Path fileToEdit, Function<Map<String, Object>, Map<String, Object>> editor) {
         Map<String, Object> filtered;
         try (InputStream inputStream = Files.newInputStream(fileToEdit)) {
@@ -384,6 +398,7 @@ protected AbstractGuideRefSetFilterDefinition(String name, GuideRef... guides) {
 
         @Override
         public void define(FilterDefinitionCollector c) {
+            c.addVersionMetadata(SAMPLED_VERSIONS);
             for (String version : SAMPLED_VERSIONS) {
                 c.addMetadata(version, guides);
                 if (QuarkusVersions.MAIN.equals(version)) {
@@ -437,6 +452,13 @@ public FilterDefinitionCollector addGuide(GuideRef ref, String version) {
             return this;
         }
 
+        public FilterDefinitionCollector addVersionMetadata(Collection<String> versions) {
+            String metadataPath = QuarkusIO.yamlVersionMetadataPath().toString();
+            addOnSourceBranch(metadataPath, metadataPath);
+            addMetadataToFilter(metadataPath, path -> yamlVersionEditor(path, versions));
+            return this;
+        }
+
         public FilterDefinitionCollector addMetadata(String version, GuideRef[] guides) {
             String metadataPath = QuarkusIO.yamlMetadataPath(version).toString();
             addOnSourceBranch(metadataPath, metadataPath);