From 611901b1e78bc135ab13fbea2498b721ec2e5d03 Mon Sep 17 00:00:00 2001 From: punAhuja Date: Tue, 11 Aug 2026 12:47:47 +0530 Subject: [PATCH 1/3] SOLR-17841: Use DocSetCollector for multithreaded DocSet search --- .../SOLR-17841-mt-docset-collector.yml | 10 ++ .../solr/search/MultiThreadedSearcher.java | 92 +++++-------------- .../apache/solr/search/SolrIndexSearcher.java | 4 +- .../search/TestMultiThreadedSearcher.java | 31 +++++++ 4 files changed, 65 insertions(+), 72 deletions(-) create mode 100644 changelog/unreleased/SOLR-17841-mt-docset-collector.yml diff --git a/changelog/unreleased/SOLR-17841-mt-docset-collector.yml b/changelog/unreleased/SOLR-17841-mt-docset-collector.yml new file mode 100644 index 000000000000..677bff81d74b --- /dev/null +++ b/changelog/unreleased/SOLR-17841-mt-docset-collector.yml @@ -0,0 +1,10 @@ +title: > + Multithreaded search now builds DocSets with DocSetCollector and skips the + parallel path when no searcher executor is configured. +type: fixed +authors: + - name: Puneet Ahuja + nick: punAhuja +links: + - name: SOLR-17841 + url: https://issues.apache.org/jira/browse/SOLR-17841 diff --git a/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java b/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java index 3ad0712ff1a9..3e36e75210eb 100644 --- a/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java @@ -22,14 +22,11 @@ import java.util.Arrays; import java.util.Collection; import java.util.Iterator; -import java.util.LinkedList; import java.util.concurrent.ExecutionException; -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.Collector; import org.apache.lucene.search.CollectorManager; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopDocsCollector; import org.apache.lucene.search.TopFieldDocs; @@ -119,11 +116,14 @@ SearchResult searchCollectorManagers( return new SearchResult(scoreMode, ret); } - static boolean allowMT(DelegatingCollector postFilter, QueryCommand cmd) { + static boolean allowMT(DelegatingCollector postFilter, QueryCommand cmd, boolean hasExecutor) { // TODO: it's unclear if segmentTerminateEarly is truly incompatible but // since it has to appropriately denote partial results this needs to be // investigated/tested before we can remove this check (perhaps for 9.8). - return postFilter == null && !cmd.getSegmentTerminateEarly() && cmd.getMultiThreaded(); + return postFilter == null + && !cmd.getSegmentTerminateEarly() + && cmd.getMultiThreaded() + && hasExecutor; } static class MaxScoreResult { @@ -134,67 +134,6 @@ public MaxScoreResult(float maxScore) { } } - static class FixedBitSetCollector extends SimpleCollector { - @SuppressWarnings("JdkObsolete") - private final LinkedList bitSets = new LinkedList<>(); - - @SuppressWarnings("JdkObsolete") - private final LinkedList skipWords = new LinkedList<>(); - - @SuppressWarnings("JdkObsolete") - private final LinkedList skipBits = new LinkedList<>(); - - FixedBitSetCollector() {} - - @Override - protected void doSetNextReader(LeafReaderContext context) throws IOException { - this.bitSets.add(null); // lazy allocate when collecting document(s) - this.skipWords.add(context.docBase / 64); - this.skipBits.add(context.docBase % 64); - } - - @Override - public void collect(int doc) throws IOException { - FixedBitSet bitSet = this.bitSets.getLast(); - final int idx = this.skipBits.getLast() + doc; - - final int numWords = FixedBitSet.bits2words(idx + 1); // +1 to ensure minimum 1 word - - if (bitSet == null) { - this.bitSets.removeLast(); - bitSet = new FixedBitSet(numWords * 64); - this.bitSets.addLast(bitSet); - - } else if (bitSet.getBits().length < numWords) { - FixedBitSet smallerBitSet = this.bitSets.removeLast(); - bitSet = new FixedBitSet(numWords * 64); - bitSet.xor(smallerBitSet); - this.bitSets.addLast(bitSet); - } - - bitSet.set(idx); - } - - void update(FixedBitSet allBitSet) { - final long[] allBits = allBitSet.getBits(); - for (int bs_idx = 0; bs_idx < this.bitSets.size(); ++bs_idx) { - final FixedBitSet itBitSet = this.bitSets.get(bs_idx); - if (itBitSet != null) { - final int skipWords = this.skipWords.get(bs_idx); - final long[] itBits = itBitSet.getBits(); - for (int idx = 0; idx < itBits.length && skipWords + idx < allBits.length; ++idx) { - allBits[skipWords + idx] ^= itBits[idx]; - } - } - } - } - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE_NO_SCORES; - } - } - static class SearchResult { final ScoreMode scoreMode; private final Object[] result; @@ -283,8 +222,7 @@ public DocSetCM(int maxDoc) { @Override public Collector newCollector() throws IOException { - // TODO: add to firstCollectors here? or if not have comment w.r.t. why not adding - return new FixedBitSetCollector(); + return new DocSetCollector(maxDoc); } @Override @@ -292,12 +230,26 @@ public Collector newCollector() throws IOException { public Object reduce(Collection collectors) throws IOException { final FixedBitSet reduced = new FixedBitSet(maxDoc); for (Object collector : collectors) { - if (collector instanceof FixedBitSetCollector fixedBitSetCollector) { - fixedBitSetCollector.update(reduced); + if (collector instanceof EarlyTerminatingCollector earlyTerminatingCollector) { + collector = earlyTerminatingCollector.getDelegate(); + } + if (collector instanceof DocSetCollector docSetCollector) { + mergeDocSetIntoFixedBitSet(docSetCollector.getDocSet(), reduced); } } return reduced; } + + private static void mergeDocSetIntoFixedBitSet(DocSet docSet, FixedBitSet reduced) { + if (docSet instanceof BitDocSet bitDocSet) { + reduced.or(bitDocSet.getBits()); + } else { + DocIterator iter = docSet.iterator(); + while (iter.hasNext()) { + reduced.set(iter.nextDoc()); + } + } + } } private class TopDocsCM implements CollectorManager { diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index 8098b39f4a19..c7bc36554e3c 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -1982,7 +1982,7 @@ public ScoreMode scoreMode() { } final TopDocs topDocs; final ScoreMode scoreModeUsed; - if (!MultiThreadedSearcher.allowMT(pf.postFilter, cmd)) { + if (!MultiThreadedSearcher.allowMT(pf.postFilter, cmd, getTaskExecutor() != null)) { log.trace("SINGLE THREADED search, skipping collector manager in getDocListNC"); final TopDocsCollector topCollector = buildTopDocsCollector(len, cmd); MaxScoreCollector maxScoreCollector = null; @@ -2093,7 +2093,7 @@ public ScoreMode scoreMode() { qr.setNextCursorMark(cmd.getCursorMark()); } else { final TopDocs topDocs; - if (!MultiThreadedSearcher.allowMT(pf.postFilter, cmd)) { + if (!MultiThreadedSearcher.allowMT(pf.postFilter, cmd, getTaskExecutor() != null)) { log.trace("SINGLE THREADED search, skipping collector manager in getDocListAndSetNC"); @SuppressWarnings({"rawtypes"}) diff --git a/solr/core/src/test/org/apache/solr/search/TestMultiThreadedSearcher.java b/solr/core/src/test/org/apache/solr/search/TestMultiThreadedSearcher.java index 2b83390158f9..54325a8730a9 100644 --- a/solr/core/src/test/org/apache/solr/search/TestMultiThreadedSearcher.java +++ b/solr/core/src/test/org/apache/solr/search/TestMultiThreadedSearcher.java @@ -123,6 +123,37 @@ public void testReRankWithMultiThreadedSearch() throws Exception { }); } + /** Multi-threaded DocSet collection must match single-threaded for the same query. */ + public void testMultiThreadedDocSetMatchesSingleThreaded() throws Exception { + h.getCore() + .withSearcher( + searcher -> { + assertTrue(searcher.getSlices().length > 1); + + final Query query = new TermQuery(new Term("field1_s", "xyzrareterm")); + final QueryCommand cmdSingle = new QueryCommand(); + cmdSingle.setQuery(query); + cmdSingle.setNeedDocSet(true); + cmdSingle.setLen(10); + cmdSingle.setMultiThreaded(false); + + final QueryCommand cmdMulti = new QueryCommand(); + cmdMulti.setQuery(query); + cmdMulti.setNeedDocSet(true); + cmdMulti.setLen(10); + cmdMulti.setMultiThreaded(true); + + final QueryResult singleThreaded = searcher.search(cmdSingle); + final QueryResult multiThreaded = searcher.search(cmdMulti); + + final DocSet stSet = singleThreaded.getDocListAndSet().docSet; + final DocSet mtSet = multiThreaded.getDocListAndSet().docSet; + assertEquals(stSet.size(), mtSet.size()); + assertTrue(DocSetUtil.equals(stSet, mtSet)); + return null; + }); + } + private static final class SimpleReRankQuery extends RankQuery { private Query q; From 6ea7cd57d182f8a06bda3cb83f03654a539fde57 Mon Sep 17 00:00:00 2001 From: Puneet Ahuja <167976912+punAhuja@users.noreply.github.com> Date: Wed, 12 Aug 2026 18:11:05 +0530 Subject: [PATCH 2/3] Update changelog/unreleased/SOLR-17841-mt-docset-collector.yml Co-authored-by: David Smiley --- changelog/unreleased/SOLR-17841-mt-docset-collector.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/changelog/unreleased/SOLR-17841-mt-docset-collector.yml b/changelog/unreleased/SOLR-17841-mt-docset-collector.yml index 677bff81d74b..0914ce58bcf3 100644 --- a/changelog/unreleased/SOLR-17841-mt-docset-collector.yml +++ b/changelog/unreleased/SOLR-17841-mt-docset-collector.yml @@ -1,6 +1,5 @@ title: > - Multithreaded search now builds DocSets with DocSetCollector and skips the - parallel path when no searcher executor is configured. + Improved multiThreaded=true performance when a docset is needed (e.g. faceting). type: fixed authors: - name: Puneet Ahuja From 500312f209a6205eb2207ba95508e20bb671e30d Mon Sep 17 00:00:00 2001 From: Puneet Ahuja <167976912+punAhuja@users.noreply.github.com> Date: Wed, 12 Aug 2026 18:11:17 +0530 Subject: [PATCH 3/3] Update changelog/unreleased/SOLR-17841-mt-docset-collector.yml Co-authored-by: David Smiley --- changelog/unreleased/SOLR-17841-mt-docset-collector.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/unreleased/SOLR-17841-mt-docset-collector.yml b/changelog/unreleased/SOLR-17841-mt-docset-collector.yml index 0914ce58bcf3..3bd48c04e885 100644 --- a/changelog/unreleased/SOLR-17841-mt-docset-collector.yml +++ b/changelog/unreleased/SOLR-17841-mt-docset-collector.yml @@ -1,6 +1,6 @@ title: > Improved multiThreaded=true performance when a docset is needed (e.g. faceting). -type: fixed +type: changed authors: - name: Puneet Ahuja nick: punAhuja