diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java index a3bbc6d1a669..1ed32cb51f00 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java @@ -178,7 +178,9 @@ private GlobalIndexResult read( * dropped, or a {@code contains} / {@code endsWith} / {@code like} leaf, which BTree * answers with every non-null row), the candidates are refined by reading their filter * columns if {@code global-index.filter.refine-from-data} allows it, and excluded - * otherwise. + * otherwise. When every index covering the filter columns declines the predicate, the + * rows are excluded in {@code fast} and otherwise decided by reading their filter + * columns. *
  • Rows whose filter columns are not covered follow {@code scalar-index.search-mode}: * excluded in {@code fast}, otherwise decided by reading their filter columns. * @@ -247,6 +249,11 @@ private RoaringNavigableMap64 matchedRows( matched.or(fromIndex); } else { warnUnindexedFilter(); + if (table.coreOptions().scalarIndexSearchMode() != GlobalIndexSearchMode.FAST) { + // No scalar index could evaluate the filter: decide these rows from the data, + // like rows whose filter columns have no index. + unindexed.or(decidedByIndex); + } } } if (!unindexed.isEmpty()) { diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java index 630c82b15d0b..7baeed2e102f 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java @@ -31,6 +31,7 @@ import org.apache.paimon.globalindex.ResultEntry; import org.apache.paimon.globalindex.ScoredGlobalIndexResult; import org.apache.paimon.globalindex.btree.BTreeGlobalIndexerFactory; +import org.apache.paimon.globalindex.fmindex.FMGlobalIndexerFactory; import org.apache.paimon.globalindex.testfulltext.TestFullTextGlobalIndexReader; import org.apache.paimon.globalindex.testfulltext.TestFullTextGlobalIndexerFactory; import org.apache.paimon.index.GlobalIndexMeta; @@ -914,6 +915,28 @@ public void testFullTextSearchCandidateOnlyIndexResultsAreNeverRanked( .containsExactly(5L); } + @ParameterizedTest(name = "scalar-index.search-mode={0}") + @CsvSource({"fast", "full", "detail"}) + public void testFullTextSearchReadsFilterColumnsWhenScalarIndexDeclinesFilter(String scalarMode) + throws Exception { + // FM covers the filter column but cannot evaluate equality, so no index decides the + // rows. Like rows without an index, full and detail decide them from the data. + FileStoreTable table = createTable("full_text_declined_" + scalarMode, scalarMode, false); + writeDocuments(table, RANKED_DOCUMENTS); + buildAndCommitIndex(table, RANKED_DOCUMENTS); + buildAndCommitBTreeIndex(table, RANKED_DOCUMENTS, FMGlobalIndexerFactory.IDENTIFIER); + + Predicate filter = + new PredicateBuilder(table.rowType()) + .equal(1, BinaryString.fromString("paimon zeta")); + RoaringNavigableMap64 rows = searchWithFilter(table, filter, 1).results(); + if (scalarMode.equals("fast")) { + assertThat(rows).isEmpty(); + } else { + assertThat(rows).containsExactly(5L); + } + } + @ParameterizedTest(name = "scalar-index.search-mode={0}, refine-from-data={1}") @CsvSource({"fast, true", "full, true", "fast, false", "full, false"}) public void testFullTextSearchPartiallyIndexedConjunctionIsNeverRankedAsSuperset( @@ -1947,16 +1970,18 @@ private void buildAndCommitIdBTreeIndexRange(FileStoreTable table, Range rowRang private void buildAndCommitBTreeIndex(FileStoreTable table, String[] documents) throws Exception { + buildAndCommitBTreeIndex(table, documents, BTreeGlobalIndexerFactory.IDENTIFIER); + } + + private void buildAndCommitBTreeIndex( + FileStoreTable table, String[] documents, String indexType) throws Exception { Options options = table.coreOptions().toConfiguration(); DataField textField = table.rowType().getField(TEXT_FIELD_NAME); GlobalIndexSingleColumnWriter writer = (GlobalIndexSingleColumnWriter) GlobalIndexBuilderUtils.createIndexWriter( - table, - BTreeGlobalIndexerFactory.IDENTIFIER, - Collections.singletonList(textField), - options); + table, indexType, Collections.singletonList(textField), options); for (int i = 0; i < documents.length; i++) { writer.write(BinaryString.fromString(documents[i]), i); } @@ -1970,7 +1995,7 @@ private void buildAndCommitBTreeIndex(FileStoreTable table, String[] documents) table.coreOptions(), rowRange, textField.id(), - BTreeGlobalIndexerFactory.IDENTIFIER, + indexType, entries); DataIncrement dataIncrement = DataIncrement.indexIncrement(indexFiles);