From 2371c464196651394ea449b7551972e0e410a2f3 Mon Sep 17 00:00:00 2001 From: Eunbin Son Date: Sun, 4 Oct 2026 13:09:07 +0900 Subject: [PATCH] [core] Decide full-text row filters from data when no scalar index can evaluate them When every scalar global index covering the filter columns declines the predicate at runtime (for example an FM index on an equality filter), DataEvolutionFullTextRead#matchedRows dropped the index-covered rows, so full-text search with a row filter returned nothing in search mode full or detail. Decide those rows by reading their filter columns, like rows whose filter columns have no index. Fast mode is unchanged. Generated-by: Claude Code --- .../source/DataEvolutionFullTextRead.java | 9 ++++- .../source/FullTextSearchBuilderTest.java | 35 ++++++++++++++++--- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java index a3bbc6d1a669..1ed32cb51f00 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java @@ -178,7 +178,9 @@ private GlobalIndexResult read( * dropped, or a {@code contains} / {@code endsWith} / {@code like} leaf, which BTree * answers with every non-null row), the candidates are refined by reading their filter * columns if {@code global-index.filter.refine-from-data} allows it, and excluded - * otherwise. + * otherwise. When every index covering the filter columns declines the predicate, the + * rows are excluded in {@code fast} and otherwise decided by reading their filter + * columns. *
  • Rows whose filter columns are not covered follow {@code scalar-index.search-mode}: * excluded in {@code fast}, otherwise decided by reading their filter columns. * @@ -247,6 +249,11 @@ private RoaringNavigableMap64 matchedRows( matched.or(fromIndex); } else { warnUnindexedFilter(); + if (table.coreOptions().scalarIndexSearchMode() != GlobalIndexSearchMode.FAST) { + // No scalar index could evaluate the filter: decide these rows from the data, + // like rows whose filter columns have no index. + unindexed.or(decidedByIndex); + } } } if (!unindexed.isEmpty()) { diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java index 630c82b15d0b..7baeed2e102f 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java @@ -31,6 +31,7 @@ import org.apache.paimon.globalindex.ResultEntry; import org.apache.paimon.globalindex.ScoredGlobalIndexResult; import org.apache.paimon.globalindex.btree.BTreeGlobalIndexerFactory; +import org.apache.paimon.globalindex.fmindex.FMGlobalIndexerFactory; import org.apache.paimon.globalindex.testfulltext.TestFullTextGlobalIndexReader; import org.apache.paimon.globalindex.testfulltext.TestFullTextGlobalIndexerFactory; import org.apache.paimon.index.GlobalIndexMeta; @@ -914,6 +915,28 @@ public void testFullTextSearchCandidateOnlyIndexResultsAreNeverRanked( .containsExactly(5L); } + @ParameterizedTest(name = "scalar-index.search-mode={0}") + @CsvSource({"fast", "full", "detail"}) + public void testFullTextSearchReadsFilterColumnsWhenScalarIndexDeclinesFilter(String scalarMode) + throws Exception { + // FM covers the filter column but cannot evaluate equality, so no index decides the + // rows. Like rows without an index, full and detail decide them from the data. + FileStoreTable table = createTable("full_text_declined_" + scalarMode, scalarMode, false); + writeDocuments(table, RANKED_DOCUMENTS); + buildAndCommitIndex(table, RANKED_DOCUMENTS); + buildAndCommitBTreeIndex(table, RANKED_DOCUMENTS, FMGlobalIndexerFactory.IDENTIFIER); + + Predicate filter = + new PredicateBuilder(table.rowType()) + .equal(1, BinaryString.fromString("paimon zeta")); + RoaringNavigableMap64 rows = searchWithFilter(table, filter, 1).results(); + if (scalarMode.equals("fast")) { + assertThat(rows).isEmpty(); + } else { + assertThat(rows).containsExactly(5L); + } + } + @ParameterizedTest(name = "scalar-index.search-mode={0}, refine-from-data={1}") @CsvSource({"fast, true", "full, true", "fast, false", "full, false"}) public void testFullTextSearchPartiallyIndexedConjunctionIsNeverRankedAsSuperset( @@ -1947,16 +1970,18 @@ private void buildAndCommitIdBTreeIndexRange(FileStoreTable table, Range rowRang private void buildAndCommitBTreeIndex(FileStoreTable table, String[] documents) throws Exception { + buildAndCommitBTreeIndex(table, documents, BTreeGlobalIndexerFactory.IDENTIFIER); + } + + private void buildAndCommitBTreeIndex( + FileStoreTable table, String[] documents, String indexType) throws Exception { Options options = table.coreOptions().toConfiguration(); DataField textField = table.rowType().getField(TEXT_FIELD_NAME); GlobalIndexSingleColumnWriter writer = (GlobalIndexSingleColumnWriter) GlobalIndexBuilderUtils.createIndexWriter( - table, - BTreeGlobalIndexerFactory.IDENTIFIER, - Collections.singletonList(textField), - options); + table, indexType, Collections.singletonList(textField), options); for (int i = 0; i < documents.length; i++) { writer.write(BinaryString.fromString(documents[i]), i); } @@ -1970,7 +1995,7 @@ private void buildAndCommitBTreeIndex(FileStoreTable table, String[] documents) table.coreOptions(), rowRange, textField.id(), - BTreeGlobalIndexerFactory.IDENTIFIER, + indexType, entries); DataIncrement dataIncrement = DataIncrement.indexIncrement(indexFiles);