[core] Decide full-text row filters from data when no scalar index can evaluate them

paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java

@@ -178,7 +178,9 @@ private GlobalIndexResult read(
      *       dropped, or a {@code contains} / {@code endsWith} / {@code like} leaf, which BTree
      *       answers with every non-null row), the candidates are refined by reading their filter
      *       columns if {@code global-index.filter.refine-from-data} allows it, and excluded
-     *       otherwise.
+     *       otherwise. When every index covering the filter columns declines the predicate, the
+     *       rows are excluded in {@code fast} and otherwise decided by reading their filter
+     *       columns.
      *   <li>Rows whose filter columns are not covered follow {@code scalar-index.search-mode}:
      *       excluded in {@code fast}, otherwise decided by reading their filter columns.
      * </ul>
@@ -247,6 +249,11 @@ private RoaringNavigableMap64 matchedRows(
                 matched.or(fromIndex);
             } else {
                 warnUnindexedFilter();
+                if (table.coreOptions().scalarIndexSearchMode() != GlobalIndexSearchMode.FAST) {
+                    // No scalar index could evaluate the filter: decide these rows from the data,
+                    // like rows whose filter columns have no index.
+                    unindexed.or(decidedByIndex);
+                }
             }
         }
         if (!unindexed.isEmpty()) {

paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java

@@ -31,6 +31,7 @@
 import org.apache.paimon.globalindex.ResultEntry;
 import org.apache.paimon.globalindex.ScoredGlobalIndexResult;
 import org.apache.paimon.globalindex.btree.BTreeGlobalIndexerFactory;
+import org.apache.paimon.globalindex.fmindex.FMGlobalIndexerFactory;
 import org.apache.paimon.globalindex.testfulltext.TestFullTextGlobalIndexReader;
 import org.apache.paimon.globalindex.testfulltext.TestFullTextGlobalIndexerFactory;
 import org.apache.paimon.index.GlobalIndexMeta;
@@ -914,6 +915,28 @@ public void testFullTextSearchCandidateOnlyIndexResultsAreNeverRanked(
                 .containsExactly(5L);
     }
 
+    @ParameterizedTest(name = "scalar-index.search-mode={0}")
+    @CsvSource({"fast", "full", "detail"})
+    public void testFullTextSearchReadsFilterColumnsWhenScalarIndexDeclinesFilter(String scalarMode)
+            throws Exception {
+        // FM covers the filter column but cannot evaluate equality, so no index decides the
+        // rows. Like rows without an index, full and detail decide them from the data.
+        FileStoreTable table = createTable("full_text_declined_" + scalarMode, scalarMode, false);
+        writeDocuments(table, RANKED_DOCUMENTS);
+        buildAndCommitIndex(table, RANKED_DOCUMENTS);
+        buildAndCommitBTreeIndex(table, RANKED_DOCUMENTS, FMGlobalIndexerFactory.IDENTIFIER);
+
+        Predicate filter =
+                new PredicateBuilder(table.rowType())
+                        .equal(1, BinaryString.fromString("paimon zeta"));
+        RoaringNavigableMap64 rows = searchWithFilter(table, filter, 1).results();
+        if (scalarMode.equals("fast")) {
+            assertThat(rows).isEmpty();
+        } else {
+            assertThat(rows).containsExactly(5L);
+        }
+    }
+
     @ParameterizedTest(name = "scalar-index.search-mode={0}, refine-from-data={1}")
     @CsvSource({"fast, true", "full, true", "fast, false", "full, false"})
     public void testFullTextSearchPartiallyIndexedConjunctionIsNeverRankedAsSuperset(
@@ -1947,16 +1970,18 @@ private void buildAndCommitIdBTreeIndexRange(FileStoreTable table, Range rowRang
 
     private void buildAndCommitBTreeIndex(FileStoreTable table, String[] documents)
             throws Exception {
+        buildAndCommitBTreeIndex(table, documents, BTreeGlobalIndexerFactory.IDENTIFIER);
+    }
+
+    private void buildAndCommitBTreeIndex(
+            FileStoreTable table, String[] documents, String indexType) throws Exception {
         Options options = table.coreOptions().toConfiguration();
         DataField textField = table.rowType().getField(TEXT_FIELD_NAME);
 
         GlobalIndexSingleColumnWriter writer =
                 (GlobalIndexSingleColumnWriter)
                         GlobalIndexBuilderUtils.createIndexWriter(
-                                table,
-                                BTreeGlobalIndexerFactory.IDENTIFIER,
-                                Collections.singletonList(textField),
-                                options);
+                                table, indexType, Collections.singletonList(textField), options);
         for (int i = 0; i < documents.length; i++) {
             writer.write(BinaryString.fromString(documents[i]), i);
         }
@@ -1970,7 +1995,7 @@ private void buildAndCommitBTreeIndex(FileStoreTable table, String[] documents)
                         table.coreOptions(),
                         rowRange,
                         textField.id(),
-                        BTreeGlobalIndexerFactory.IDENTIFIER,
+                        indexType,
                         entries);
 
         DataIncrement dataIncrement = DataIncrement.indexIncrement(indexFiles);