Merge branch 'main' of https://github.com/apache/lucene into deprecated_usage

Author: slow-J

.gitattributes

@@ -1,6 +1,6 @@
 # Ignore all differences in line endings for the lock file.
-versions.lock text eol=lf
-versions.props text eol=lf
+versions.lock  text eol=lf
+versions.toml  text eol=lf
 
 # Gradle files are always in LF.
 *.gradle          text eol=lf

README.md

@@ -23,6 +23,7 @@ Apache Lucene is a high-performance, full-featured text search engine library
 written in Java.
 
 [![Build Status](https://ci-builds.apache.org/job/Lucene/job/Lucene-Artifacts-main/badge/icon?subject=Lucene)](https://ci-builds.apache.org/job/Lucene/job/Lucene-Artifacts-main/)
+[![Revved up by Develocity](https://img.shields.io/badge/Revved%20up%20by-Develocity-06A0CE?logo=Gradle&labelColor=02303A)](https://ge.apache.org/scans?search.buildToolType=gradle&search.rootProjectNames=lucene-root)
 
 ## Online Documentation
 

dev-tools/doap/lucene.rdf

@@ -67,6 +67,13 @@
     </maintainer>
 
     <!-- NOTE: please insert releases in numeric order, NOT chronologically. -->
+    <release>
+       <Version>
+         <name>lucene-9.11.1</name>
+         <created>2024-06-27</created>
+         <revision>9.11.1</revision>
+       </Version>
+    </release>. 
     <release>
        <Version>
          <name>lucene-9.11.0</name>

dev-tools/scripts/diff_lucene_changes.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import urllib.request
+
+'''
+A simple tool to see diffs between main's version of CHANGES.txt entries for
+a given release vs the stable branch's version.  It's best to keep these 1)
+identical and 2) matching what changes were actually backported to be honest
+to users and avoid future annoying conflicts on backport.
+'''
+
+# e.g. python3 -u diff_lucene_changes.py branch_9_9 main 9.9.0
+
+#
+
+def get_changes_url(branch_name):
+  if os.path.isdir(branch_name):
+    url = f'file://{branch_name}/lucene/CHANGES.txt'
+  else:
+    url = f'https://raw.githubusercontent.com/apache/lucene/{branch_name}/lucene/CHANGES.txt'
+  print(f'NOTE: resolving {branch_name} --> {url}')
+  return url
+
+def extract_release_section(changes_txt, release_name):
+  return re.search(f'=======+ Lucene {re.escape(release_name)} =======+(.*?)=======+ Lucene .*? =======+$',
+                   changes_txt.decode('utf-8'), re.MULTILINE | re.DOTALL).group(1).encode('utf-8')
+
+def main():
+  if len(sys.argv) < 3 or len(sys.argv) > 5:
+    print('\nUsage: python3 -u dev-tools/scripts/diff_lucene_changes.py <branch1-or-local-clone> <branch2-or-local-clone> <release-name> [diff-commandline-extras]\n')
+    print('  e.g.: python3 -u dev-tools/scripts/diff_lucene_changes.py branch_9_9 /l/trunk 9.9.0 "-w"\n')
+    sys.exit(1)
+
+  branch1 = sys.argv[1]
+  branch2 = sys.argv[2]
+  release_name = sys.argv[3]
+
+  if len(sys.argv) > 4:
+    diff_cl_extras = [sys.argv[4]]
+  else:
+    diff_cl_extras = []
+
+  branch1_changes = extract_release_section(urllib.request.urlopen(get_changes_url(branch1)).read(),
+                                            release_name)
+  branch2_changes = extract_release_section(urllib.request.urlopen(get_changes_url(branch2)).read(),
+                                            release_name)
+
+  with tempfile.NamedTemporaryFile() as f1, tempfile.NamedTemporaryFile() as f2:
+    f1.write(branch1_changes)
+    f2.write(branch2_changes)
+
+    command = ['diff'] + diff_cl_extras + [f1.name, f2.name]
+
+    # diff returns non-zero exit status when there are diffs, so don't pass check=True
+    print(subprocess.run(command, check=False, capture_output=True).stdout.decode('utf-8'))
+
+if __name__ == '__main__':
+  main()

lucene/CHANGES.txt

@@ -80,10 +80,6 @@ API Changes
 * GITHUB#12875: Ensure token position is always increased in PathHierarchyTokenizer and ReversePathHierarchyTokenizer
   and resulting tokens do not overlap. (Michael Froh, Lukáš Vlček)
 
-* GITHUB#12624, GITHUB#12831: Allow FSTCompiler to stream to any DataOutput while building, and
-  make compile() only return the FSTMetadata. For on-heap (default) use case, please use
-  FST.fromFSTReader(fstMetadata, fstCompiler.getFSTReader()) to create the FST. (Anh Dung Bui)
-
 * GITHUB#13146, GITHUB#13148: Remove ByteBufferIndexInput and only use MemorySegment APIs
   for MMapDirectory.  (Uwe Schindler)
 
@@ -133,6 +129,11 @@ New Features
   DocValuesSkipper abstraction. A new flag is added to FieldType.java that configures whether
   to create a "skip index" for doc values. (Ignacio Vera)
 
+* GITHUB#13563: Add levels to doc values skip index. (Ignacio Vera)
+
+* GITHUB#13597: Align doc value skipper interval boundaries when an interval contains a constant
+  value. (Ignacio Vera)
+
 Improvements
 ---------------------
 
@@ -256,11 +257,23 @@ API Changes
 
 New Features
 ---------------------
-(No changes)
+
+* GITHUB#13430: Allow configuring the search concurrency via
+  TieredMergePolicy#setTargetSearchConcurrency. This in-turn instructs the
+  merge policy to try to have at least this number of segments on the highest
+  tier. (Adrien Grand, Carlos Delgado)
+
+* GITHUB#13517: Allow configuring the search concurrency on LogDocMergePolicy
+  and LogByteSizeMergePolicy via a new #setTargetConcurrency setter.
+  (Adrien Grand)
 
 Improvements
 ---------------------
-(No changes)
+
+* GITHUB#13548: Refactor and javadoc update for KNN vector writer classes. (Patrick Zhai)
+
+* GITHUB#13562: Add Intervals.regexp and Intervals.range methods to produce  IntervalsSource
+  for regexp and range queries. (Mayya Sharipova)
 
 Optimizations
 ---------------------
@@ -279,14 +292,65 @@ Optimizations
 
 * GITHUB#12941: Don't preserve auxiliary buffer contents in LSBRadixSorter if it grows. (Stefan Vodita)
 
+* GITHUB#13175: Stop double-checking priority queue inserts in some FacetCount classes. (Jakub Slowinski)
+
+* GITHUB#13538: Slightly reduce heap usage for HNSW and scalar quantized vector writers. (Ben Trent)
+
+* GITHUB#12100: WordBreakSpellChecker.suggestWordBreaks now does a breadth first search, allowing it to return
+  better matches with fewer evaluations (hossman)
+
+* GITHUB#13582: Stop requiring MaxScoreBulkScorer's outer window from having at
+  least INNER_WINDOW_SIZE docs. (Adrien Grand)
+
+* GITHUB#13570, GITHUB#13574, GITHUB#13535: Avoid performance degradation with closing shared Arenas.
+  Closing many individual index files can potentially lead to a degradation in execution performance.
+  Index files are mmapped one-to-one with the JDK's foreign shared Arena. The JVM deoptimizes the top
+  few frames of all threads when closing a shared Arena (see JDK-8335480). We mitigate this situation
+  by 1) using a confined Arena where appropriate, and 2) grouping files from the same segment to a
+  single shared Arena. (Chris Hegarty, Michael Gibney, Uwe Schindler)
+
+Changes in runtime behavior
+---------------------
+
+* GITHUB#13472: When an executor is provided to the IndexSearcher constructor, the searcher now executes tasks on the
+  thread that invoked a search as well as its configured executor. Users should reduce the executor's thread-count by 1
+  to retain the previous level of parallelism. Moreover, it is now possible to start searches from the same executor
+  that is configured in the IndexSearcher without risk of deadlocking. A separate executor for starting searches is no
+  longer required. (Armin Braun)
+
 Bug Fixes
 ---------------------
-(No changes)
+
+* GITHUB#13384: Fix highlighter to use longer passages instead of shorter individual terms. (Zack Kendall)
+
+* GITHUB#13463: Address bug in MultiLeafKnnCollector causing #minCompetitiveSimilarity to stay artificially low in
+  some corner cases. (Greg Miller)
+
+* GITHUB#13553: Correct RamUsageEstimate for scalar quantized knn vector formats so that raw vectors are correctly
+  accounted for. (Ben Trent)
 
 Other
 --------------------
 (No changes)
 
+======================== Lucene 9.11.1 =======================
+
+Bug Fixes
+---------------------
+
+* GITHUB#13498: Avoid performance regression by constructing lazily the PointTree in NumericComparator. (Ignacio Vera)
+
+* GITHUB#13501, GITHUB#13478: Remove intra-merge parallelism for everything except HNSW graph merges. (Ben Trent)
+
+* GITHUB#13498, GITHUB#13340: Allow adding a parent field to an index with no fields (Michael Sokolov)
+
+* GITHUB#12431: Fix IndexOutOfBoundsException thrown in DefaultPassageFormatter
+  by unordered matches. (Stephane Campinas)
+
+* GITHUB#13493: StringValueFacetCounts stops throwing NPE when faceting over an empty match-set. (Grebennikov Roman,
+  Stefan Vodita)
+
+
 ======================== Lucene 9.11.0 =======================
 
 API Changes
@@ -494,13 +558,23 @@ API Changes
 
 * GITHUB#12854: Mark DrillSideways#createDrillDownFacetsCollector as @Deprecated. (Greg Miller)
 
+* GITHUB#12624, GITHUB#12831: Allow FSTCompiler to stream to any DataOutput while building, and
+  make compile() only return the FSTMetadata. For on-heap (default) use case, please use
+  FST.fromFSTReader(fstMetadata, fstCompiler.getFSTReader()) to create the FST. (Anh Dung Bui)
+
 New Features
 ---------------------
 * GITHUB#12679: Add support for similarity-based vector searches using [Byte|Float]VectorSimilarityQuery. Uses a new
   VectorSimilarityCollector to find all vectors scoring above a `resultSimilarity` while traversing the HNSW graph till
   better-scoring nodes are available, or the best candidate is below a score of `traversalSimilarity` in the lowest
   level. (Aditya Prakash, Kaival Parikh)
 
+* GITHUB#12829: For indices newly created as of 9.10.0 onwards, IndexWriter preserves document blocks indexed via
+  IndexWriter#addDocuments or IndexWriter#updateDocuments also when index sorting is configured. Document blocks are
+  maintained alongside their parent documents during sort and merge. IndexWriterConfig accepts a parent field that is used
+  to maintain block orders if index sorting is used. Note, this is fully optional in Lucene 9.x while will be mandatory for
+  indices that use document blocks together with index sorting as of 10.0.0. (Simon Willnauer)
+
 * GITHUB#12336: Index additional data per facet label in the taxonomy. (Shai Erera, Egor Potemkin, Mike McCandless,
   Stefan Vodita)
 
@@ -592,7 +666,6 @@ Build
 
 Other
 ---------------------
-
 * GITHUB#11023: Removing some dead code in CheckIndex. (Jakub Slowinski)
 
 * GITHUB#11023: Removing @lucene.experimental tags in testXXX methods in CheckIndex. (Jakub Slowinski)

lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene40/blocktree/FieldReader.java

@@ -88,21 +88,17 @@ public final class FieldReader extends Terms {
         (new ByteArrayDataInput(rootCode.bytes, rootCode.offset, rootCode.length)).readVLong()
             >>> Lucene40BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS;
     // Initialize FST always off-heap.
-    final IndexInput clone = indexIn.clone();
-    clone.seek(indexStartFP);
+    final FST.FSTMetadata<BytesRef> fstMetadata;
     if (metaIn == indexIn) { // Only true before Lucene 8.6
-      index =
-          new FST<>(
-              readMetadata(clone, ByteSequenceOutputs.getSingleton()),
-              clone,
-              new OffHeapFSTStore());
+      final IndexInput clone = indexIn.clone();
+      clone.seek(indexStartFP);
+      fstMetadata = readMetadata(clone, ByteSequenceOutputs.getSingleton());
+      // FST bytes actually only start after the metadata.
+      indexStartFP = clone.getFilePointer();
     } else {
-      index =
-          new FST<>(
-              readMetadata(metaIn, ByteSequenceOutputs.getSingleton()),
-              clone,
-              new OffHeapFSTStore());
+      fstMetadata = readMetadata(metaIn, ByteSequenceOutputs.getSingleton());
     }
+    index = FST.fromFSTReader(fstMetadata, new OffHeapFSTStore(indexIn, indexStartFP, fstMetadata));
     /*
      if (false) {
      final String dotFileName = segment + "_" + fieldInfo.name + ".dot";