diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/CompositeKeySerializer.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/CompositeKeySerializer.java index 0a5f6a294486..8b9ac9027639 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/CompositeKeySerializer.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/CompositeKeySerializer.java @@ -20,87 +20,56 @@ import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.data.serializer.RowCompactedSerializer; import org.apache.paimon.memory.MemorySlice; -import org.apache.paimon.memory.MemorySliceInput; -import org.apache.paimon.memory.MemorySliceOutput; import org.apache.paimon.types.RowType; import java.util.Comparator; -/** Length-delimited tuple keys, compared by their typed components with nulls first. */ +/** Compacted row keys, compared by their typed components with nulls first. */ public class CompositeKeySerializer implements KeySerializer { - private final RowType rowType; - - private final KeySerializer[] serializers; + private final ThreadLocal serializer; private final InternalRow.FieldGetter[] getters; private final Comparator[] comparators; @SuppressWarnings("unchecked") public CompositeKeySerializer(RowType type) { - this.rowType = type; + serializer = ThreadLocal.withInitial(() -> new RowCompactedSerializer(type)); int count = type.getFieldCount(); - serializers = new KeySerializer[count]; getters = new InternalRow.FieldGetter[count]; comparators = new Comparator[count]; for (int i = 0; i < count; i++) { - serializers[i] = KeySerializer.create(type.getTypeAt(i)); getters[i] = InternalRow.createFieldGetter(type.getTypeAt(i), i); - comparators[i] = serializers[i].createComparator(); + comparators[i] = KeySerializer.create(type.getTypeAt(i)).createComparator(); } } - public RowType rowType() { - return rowType; - } - @Override public byte[] serialize(Object key) { - InternalRow row = (InternalRow) key; - if (row.getFieldCount() != serializers.length) { - throw new IllegalArgumentException( - "Expected " - + serializers.length - + " composite key fields, but got " - + row.getFieldCount()); - } - MemorySliceOutput output = new MemorySliceOutput(32); - for (int i = 0; i < serializers.length; i++) { - Object value = getters[i].getFieldOrNull(row); - if (value == null) { - output.writeInt(-1); - } else { - byte[] bytes = serializers[i].serialize(value); - output.writeInt(bytes.length); - output.writeBytes(bytes); + // Canonicalize RowKind and NaN payloads so equal keys have the same Bloom hash. + GenericRow row = new GenericRow(getters.length); + for (int i = 0; i < getters.length; i++) { + Object value = getters[i].getFieldOrNull((InternalRow) key); + if (value instanceof Float && Float.isNaN((Float) value)) { + value = Float.NaN; + } else if (value instanceof Double && Double.isNaN((Double) value)) { + value = Double.NaN; } + row.setField(i, value); } - return output.toSlice().copyBytes(); + return serializer.get().serializeToBytes(row); } @Override public Object deserialize(MemorySlice data) { - MemorySliceInput input = data.toInput(); - GenericRow row = new GenericRow(serializers.length); - for (int i = 0; i < serializers.length; i++) { - int length = input.readInt(); - if (length >= 0) { - row.setField(i, serializers[i].deserialize(input.readSlice(length))); - } else if (length != -1) { - throw new IllegalArgumentException( - "Invalid composite key component length: " + length); - } - } - if (input.available() != 0) { - throw new IllegalArgumentException("Trailing bytes in composite key"); - } - return row; + return serializer.get().deserialize(data.copyBytes()); } @Override public Comparator createComparator() { return (left, right) -> { - for (int i = 0; i < serializers.length; i++) { + for (int i = 0; i < getters.length; i++) { Object a = getters[i].getFieldOrNull((InternalRow) left); Object b = getters[i].getFieldOrNull((InternalRow) right); int comparison = diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/ContainsRefinementEvaluator.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/ContainsRefinementEvaluator.java index 8e4c74a44244..a5e138a11b1c 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/ContainsRefinementEvaluator.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/ContainsRefinementEvaluator.java @@ -131,7 +131,8 @@ CompletableFuture> evaluate( Optional.of( new GlobalIndexEvaluator.Evaluation( GlobalIndexResult.createEmpty(), - emptyResultFields))); + emptyResultFields, + null))); } } List groupedContains = new ArrayList<>(groups.values()); @@ -169,7 +170,8 @@ CompletableFuture> evaluate( .Evaluation( GlobalIndexResult .createEmpty(), - emptyResultFields))); + emptyResultFields, + null))); } } @@ -269,7 +271,8 @@ private CompletableFuture> visitContai value -> new GlobalIndexEvaluator.Evaluation( value, - Collections.singleton(group.fieldId)))); + Collections.singleton(group.fieldId), + null))); } private CompletableFuture> intersectReaderResults( diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexEvaluator.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexEvaluator.java index b772cc7edbe9..d43193aae0a8 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexEvaluator.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexEvaluator.java @@ -37,6 +37,7 @@ import org.apache.paimon.predicate.TopN; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.IOUtils; +import org.apache.paimon.utils.Range; import javax.annotation.Nullable; @@ -188,7 +189,8 @@ private CompletableFuture> visitFieldAsync( } return compoundResult.map( result -> - new Evaluation(result, Collections.singleton(fieldId))); + new Evaluation( + result, Collections.singleton(fieldId), null)); }); } @@ -278,7 +280,7 @@ private Optional combineResults( compoundResult = compoundResult.or(child.get().result()); contributingFieldIds.addAll(child.get().contributingFieldIds()); } - return Optional.of(new Evaluation(compoundResult, contributingFieldIds)); + return Optional.of(new Evaluation(compoundResult, contributingFieldIds, null)); } else { Optional compoundResult = Optional.empty(); for (Optional child : results) { @@ -295,7 +297,7 @@ private Optional combineResults( break; } } - return compoundResult.map(result -> new Evaluation(result, contributingFieldIds)); + return compoundResult.map(result -> new Evaluation(result, contributingFieldIds, null)); } } @@ -306,9 +308,17 @@ public static final class Evaluation { private final GlobalIndexResult result; private final Set contributingFieldIds; + @Nullable private final List coveredRanges; - Evaluation(GlobalIndexResult result, Collection contributingFieldIds) { + Evaluation( + GlobalIndexResult result, + Collection contributingFieldIds, + @Nullable List coveredRanges) { this.result = result; + this.coveredRanges = + coveredRanges == null + ? null + : Collections.unmodifiableList(new ArrayList<>(coveredRanges)); this.contributingFieldIds = Collections.unmodifiableSet(new HashSet<>(contributingFieldIds)); } @@ -317,6 +327,11 @@ public GlobalIndexResult result() { return result; } + @Nullable + public List coveredRanges() { + return coveredRanges; + } + public Set contributingFieldIds() { return contributingFieldIds; } diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexReader.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexReader.java index 4506713b9176..537ca1f0bf00 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexReader.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexReader.java @@ -23,6 +23,7 @@ import org.apache.paimon.predicate.FullTextSearch; import org.apache.paimon.predicate.FunctionVisitor; import org.apache.paimon.predicate.LeafPredicate; +import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.TopN; import org.apache.paimon.predicate.VectorSearch; @@ -36,9 +37,8 @@ public interface GlobalIndexReader extends FunctionVisitor>>, Closeable { - /** Point lookup of a full composite key, with literals in index column order. */ - default CompletableFuture> visitCompositeEqual( - List literals) { + /** Evaluate a composite predicate. An empty Optional means the reader cannot use the index. */ + default CompletableFuture> visitComposite(Predicate predicate) { return CompletableFuture.completedFuture(Optional.empty()); } diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexer.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexer.java index 451b292f4c2c..e8db64056d7c 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexer.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexer.java @@ -41,6 +41,7 @@ import javax.annotation.Nullable; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.ExecutorService; @@ -74,12 +75,14 @@ public class BTreeGlobalIndexer implements SortedGlobalIndexer { private static final double BLOOM_FILTER_FPP = 0.05; private final KeySerializer keySerializer; + private final List fields; private final GlobalIndexKeyExtractor keyExtractor; private final Options options; private final long fallbackScanMaxSize; private final LazyField cacheManager; public BTreeGlobalIndexer(List fields, Options options) { + this.fields = new ArrayList<>(fields); checkArgument(!fields.isEmpty(), "BTree index requires at least one field."); for (DataField field : fields) { if (field.type() instanceof RowType) { @@ -140,6 +143,7 @@ public GlobalIndexReader createReader( ExecutorService executor) { return new LazyFilteredBTreeReader( files, + fields, keySerializer, fileReader, cacheManager.get(), diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexerFactory.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexerFactory.java index d0b1140fbad0..75ae2a762bd7 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexerFactory.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexerFactory.java @@ -18,16 +18,23 @@ package org.apache.paimon.globalindex.btree; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.globalindex.CompositeKeySerializer; import org.apache.paimon.globalindex.GlobalIndexIOMeta; import org.apache.paimon.globalindex.GlobalIndexer; import org.apache.paimon.globalindex.GlobalIndexerFactory; import org.apache.paimon.globalindex.KeySerializer; import org.apache.paimon.globalindex.SortedFileMetaSelector; import org.apache.paimon.options.Options; +import org.apache.paimon.predicate.FieldRef; +import org.apache.paimon.predicate.LeafPredicate; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.types.DataField; +import org.apache.paimon.types.RowType; +import java.util.Collections; import java.util.List; +import java.util.Optional; /** The {@link GlobalIndexerFactory} for btree index. */ public class BTreeGlobalIndexerFactory implements GlobalIndexerFactory { @@ -43,8 +50,25 @@ public String identifier() { public List selectFiles( List indexFields, Predicate predicate, List files) { if (indexFields.size() > 1) { - // Scalar predicates cannot safely prune tuple metadata. - return files; + Optional> matched = + CompositeBTreePredicate.match(indexFields, predicate); + if (!matched.isPresent()) { + return files; + } + if (CompositeBTreePredicate.isContradictory(indexFields, predicate)) { + return Collections.emptyList(); + } + if (files.stream().anyMatch(file -> file.metadata() == null)) { + return files; + } + RowType keyType = new RowType(indexFields); + KeySerializer serializer = new CompositeKeySerializer(keyType); + Object[] values = matched.get().stream().map(leaf -> leaf.literals().get(0)).toArray(); + return new SortedFileMetaSelector(files, serializer) + .visitEqual( + new FieldRef(0, indexFields.get(0).name(), keyType), + GenericRow.of(values)) + .orElse(files); } return SortedFileMetaSelector.selectFiles( predicate, files, KeySerializer.create(indexFields.get(0).type())); diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexWriter.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexWriter.java index 238e70df4166..fecf600f56ac 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexWriter.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexWriter.java @@ -20,7 +20,6 @@ import org.apache.paimon.compression.BlockCompressionFactory; import org.apache.paimon.fs.PositionOutputStream; -import org.apache.paimon.globalindex.CompositeKeySerializer; import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; import org.apache.paimon.globalindex.KeySerializer; import org.apache.paimon.globalindex.ResultEntry; @@ -149,27 +148,19 @@ public void write(@Nullable Object key, long rowId) { return; } - boolean differentKey = lastKey == null || comparator.compare(key, lastKey) != 0; - if (lastKey != null && differentKey) { + if (lastKey != null && comparator.compare(key, lastKey) != 0) { try { flush(); } catch (IOException e) { throw new RuntimeException("Error in writing btree index files.", e); } } - if (differentKey || !(keySerializer instanceof CompositeKeySerializer)) { - // Sorted engine iterators can reuse the row backing a tuple key. - lastKey = - keySerializer instanceof CompositeKeySerializer - ? keySerializer.deserialize( - MemorySlice.wrap(keySerializer.serialize(key))) - : key; - } + lastKey = key; currentRowIds.add(rowId); // update stats if (firstKey == null) { - firstKey = lastKey; + firstKey = key; } } diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/CompositeBTreePredicate.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/CompositeBTreePredicate.java new file mode 100644 index 000000000000..02b71aec8909 --- /dev/null +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/CompositeBTreePredicate.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.globalindex.btree; + +import org.apache.paimon.globalindex.KeySerializer; +import org.apache.paimon.predicate.Equal; +import org.apache.paimon.predicate.LeafPredicate; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.PredicateBuilder; +import org.apache.paimon.types.DataField; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; + +/** Matches a full conjunction of equalities in physical composite-key order. */ +public final class CompositeBTreePredicate { + + private CompositeBTreePredicate() {} + + /** SQL equalities with NULL, or distinct values for the same key field, cannot match. */ + public static boolean isContradictory(List fields, Predicate predicate) { + List matched = match(fields, predicate).get(); + for (int i = 0; i < fields.size(); i++) { + Object value = matched.get(i).literals().get(0); + if (value == null) { + return true; + } + Comparator comparator = + KeySerializer.create(fields.get(i).type()).createComparator(); + for (Predicate child : PredicateBuilder.splitAnd(predicate)) { + if (child instanceof LeafPredicate) { + LeafPredicate leaf = (LeafPredicate) child; + if (leaf.function() instanceof Equal + && leaf.fieldRefOptional().equals(matched.get(i).fieldRefOptional())) { + Object other = leaf.literals().get(0); + if (other == null || comparator.compare(value, other) != 0) { + return true; + } + } + } + } + } + return false; + } + + public static Optional> match(List fields, Predicate predicate) { + List conjuncts = PredicateBuilder.splitAnd(predicate); + List matched = new ArrayList<>(); + for (DataField field : fields) { + LeafPredicate equality = null; + for (Predicate conjunct : conjuncts) { + if (conjunct instanceof LeafPredicate) { + LeafPredicate leaf = (LeafPredicate) conjunct; + if (leaf.function() instanceof Equal + && leaf.fieldRefOptional().isPresent() + && field.name().equals(leaf.fieldRefOptional().get().name()) + && field.type().equalsIgnoreNullable(leaf.type())) { + equality = leaf; + break; + } + } + } + if (equality == null) { + return Optional.empty(); + } + matched.add(equality); + } + return Optional.of(matched); + } +} diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/LazyFilteredBTreeReader.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/LazyFilteredBTreeReader.java index baadcc3c7119..b14cf376c5b9 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/LazyFilteredBTreeReader.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/LazyFilteredBTreeReader.java @@ -19,7 +19,6 @@ package org.apache.paimon.globalindex.btree; import org.apache.paimon.data.GenericRow; -import org.apache.paimon.globalindex.CompositeKeySerializer; import org.apache.paimon.globalindex.GlobalIndexIOMeta; import org.apache.paimon.globalindex.GlobalIndexResult; import org.apache.paimon.globalindex.KeySerializer; @@ -29,7 +28,10 @@ import org.apache.paimon.io.cache.CacheManager; import org.apache.paimon.memory.MemorySlice; import org.apache.paimon.predicate.FieldRef; +import org.apache.paimon.predicate.LeafPredicate; +import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.TopN; +import org.apache.paimon.types.DataField; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.Range; import org.apache.paimon.utils.RoaringNavigableMap64; @@ -39,11 +41,9 @@ import java.io.IOException; import java.util.Comparator; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; -import java.util.function.Predicate; import java.util.function.Supplier; /** @@ -54,6 +54,7 @@ public class LazyFilteredBTreeReader extends SortedFileGlobalIndexReader { private final KeySerializer keySerializer; + private final List indexFields; private final CacheManager cacheManager; private final GlobalIndexFileReader fileReader; @Nullable private final RoaringNavigableMap64 rowIdFilter; @@ -63,6 +64,7 @@ public class LazyFilteredBTreeReader extends SortedFileGlobalIndexReader files, + List indexFields, KeySerializer keySerializer, GlobalIndexFileReader fileReader, CacheManager cacheManager, @@ -74,6 +76,7 @@ public LazyFilteredBTreeReader( this.cacheManager = cacheManager; this.fileReader = fileReader; this.keySerializer = keySerializer; + this.indexFields = indexFields; this.rowIdFilter = rowRanges == null ? null : GlobalIndexResult.fromRanges(rowRanges).results(); this.comparator = keySerializer.createComparator(); @@ -113,20 +116,20 @@ private Pair fullRangeBounds(List files) { } @Override - public CompletableFuture> visitCompositeEqual( - List literals) { - if (!(keySerializer instanceof CompositeKeySerializer)) { + public CompletableFuture> visitComposite(Predicate predicate) { + if (indexFields.size() < 2) { return CompletableFuture.completedFuture(Optional.empty()); } - int fieldCount = ((CompositeKeySerializer) keySerializer).rowType().getFieldCount(); - if (literals.size() != fieldCount) { - throw new IllegalArgumentException( - "Expected " + fieldCount + " composite key fields, but got " + literals.size()); + Optional> matched = + CompositeBTreePredicate.match(indexFields, predicate); + if (!matched.isPresent()) { + return CompletableFuture.completedFuture(Optional.empty()); } - if (literals.stream().anyMatch(Objects::isNull)) { + if (CompositeBTreePredicate.isContradictory(indexFields, predicate)) { return CompletableFuture.completedFuture(Optional.of(GlobalIndexResult.createEmpty())); } - return visitEqual((FieldRef) null, GenericRow.of(literals.toArray())); + Object[] values = matched.get().stream().map(leaf -> leaf.literals().get(0)).toArray(); + return visitEqual((FieldRef) null, GenericRow.of(values)); } @Override @@ -183,7 +186,7 @@ public CompletableFuture> visitBetween( // Only use for predicates whose matching keys form one contiguous interval. private CompletableFuture> visitWithAllMatch( - Predicate predicate, + java.util.function.Predicate predicate, Supplier>> fallback) { if (fullRangeBounds != null && predicate.test(fullRangeBounds.getLeft()) diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeBloomFilterTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeBloomFilterTest.java index 1db4a2c22296..dd72b0ea22e2 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeBloomFilterTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeBloomFilterTest.java @@ -18,9 +18,11 @@ package org.apache.paimon.globalindex.btree; +import org.apache.paimon.data.GenericRow; import org.apache.paimon.fs.ByteArraySeekableStream; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.PositionOutputStream; +import org.apache.paimon.globalindex.CompositeKeySerializer; import org.apache.paimon.globalindex.GlobalIndexIOMeta; import org.apache.paimon.globalindex.GlobalIndexResult; import org.apache.paimon.globalindex.GlobalIndexSingleColumnWriter; @@ -34,9 +36,12 @@ import org.apache.paimon.options.Options; import org.apache.paimon.sst.BloomFilterHandle; import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.IntType; +import org.apache.paimon.types.RowType; import org.apache.paimon.utils.BloomFilter; import org.apache.paimon.utils.MurmurHashUtils; +import org.apache.paimon.utils.Range; import org.junit.jupiter.api.Test; @@ -75,6 +80,43 @@ void testBloomFilterIsNotWrittenByDefault() throws Exception { assertThat(fixture.footer.getBloomFilterHandle()).isNull(); } + @Test + void testCompositeNaNPointLookupWithBloom() throws Exception { + RowType type = RowType.of(DataTypes.FLOAT(), DataTypes.DOUBLE()); + Options options = new Options(); + options.set(BTreeIndexOptions.BTREE_INDEX_BLOOM_FILTER_ENABLED, true); + ByteArrayGlobalIndexFileWriter fileWriter = new ByteArrayGlobalIndexFileWriter(); + GlobalIndexSingleColumnWriter writer = + new BTreeGlobalIndexer(type.getFields(), options).createWriter(fileWriter); + writer.write(GenericRow.of(-1.0f, -1.0d), 0); + writer.write( + GenericRow.of( + Float.intBitsToFloat(0xffc00001), + Double.longBitsToDouble(0xfff8000000000001L)), + 1); + writer.write(GenericRow.of(Float.NaN, Double.NaN), 2); + ResultEntry entry = writer.finish().get(0); + byte[] bytes = fileWriter.bytes(); + GlobalIndexIOMeta meta = + new GlobalIndexIOMeta( + new Path(entry.fileName()), bytes.length, entry.rowCount(), entry.meta()); + try (CacheManager cacheManager = new CacheManager(MemorySize.ofMebiBytes(1), 0.5); + BTreeIndexReader reader = + new BTreeIndexReader( + new CompositeKeySerializer(type), + ignored -> new ByteArraySeekableStream(bytes), + meta, + cacheManager, + null)) { + assertThat( + reader.visitEqual(GenericRow.of(Float.NaN, Double.NaN)) + .get() + .results() + .toRangeList()) + .containsExactly(new Range(1, 2)); + } + } + @Test void testBloomFilterAvoidsDataReadForMissingLookups() throws Exception { Options options = new Options(); diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/CompositeBTreeIndexTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/CompositeBTreeIndexTest.java index 7a75de7382fa..1c05959f13a0 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/CompositeBTreeIndexTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/CompositeBTreeIndexTest.java @@ -20,6 +20,7 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.serializer.RowCompactedSerializer; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.PositionOutputStream; import org.apache.paimon.fs.local.LocalFileIO; @@ -31,12 +32,15 @@ import org.apache.paimon.globalindex.KeySerializer; import org.apache.paimon.globalindex.ResultEntry; import org.apache.paimon.globalindex.SortedGlobalIndexer; +import org.apache.paimon.globalindex.SortedIndexFileMeta; import org.apache.paimon.globalindex.io.GlobalIndexFileWriter; import org.apache.paimon.memory.MemorySlice; import org.apache.paimon.options.Options; +import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowKind; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.Range; @@ -45,12 +49,17 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.UUID; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import static org.apache.paimon.shade.guava30.com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; import static org.assertj.core.api.Assertions.assertThat; @@ -63,7 +72,7 @@ class CompositeBTreeIndexTest { @ParameterizedTest @ValueSource(ints = {1, 2}) - void testMutableTupleKeysPostingListsAndLocalRanges(int version) throws Exception { + void testDeserializedTupleKeysPostingListsAndLocalRanges(int version) throws Exception { RowType type = new RowType( Arrays.asList( @@ -91,17 +100,28 @@ public PositionOutputStream newOutputStream(String name) } }; GenericRow reused = row("category-a", -1, ""); - GlobalIndexSingleColumnWriter writer = - (GlobalIndexSingleColumnWriter) indexer.createWriter(files); - writer.write(reused, 0); + reused.setRowKind(RowKind.UPDATE_AFTER); + KeySerializer serializer = new CompositeKeySerializer(type); + List keys = new ArrayList<>(); + keys.add(serializer.serialize(reused)); reused.setField(1, 7); - writer.write(reused, 1); + keys.add(serializer.serialize(reused)); reused.setField(2, BinaryString.fromString("tag")); - writer.write(reused, 2); - writer.write(reused, 3); + keys.add(serializer.serialize(reused)); + keys.add(serializer.serialize(reused)); reused.setField(0, BinaryString.fromString("category-b")); - writer.write(reused, 4); + keys.add(serializer.serialize(reused)); + GlobalIndexSingleColumnWriter writer = + (GlobalIndexSingleColumnWriter) indexer.createWriter(files); + for (int i = 0; i < keys.size(); i++) { + writer.write(serializer.deserialize(MemorySlice.wrap(keys.get(i))), i); + } ResultEntry result = writer.finish().get(0); + SortedIndexFileMeta indexMeta = SortedIndexFileMeta.deserialize(result.meta()); + assertThat(serializer.deserialize(MemorySlice.wrap(indexMeta.getFirstKey()))) + .isEqualTo(row("category-a", -1, "")); + assertThat(serializer.deserialize(MemorySlice.wrap(indexMeta.getLastKey()))) + .isEqualTo(row("category-b", 7, "tag")); Path path = new Path(directory, result.fileName()); GlobalIndexIOMeta meta = new GlobalIndexIOMeta(path, io.getFileSize(path), result.rowCount(), result.meta()); @@ -121,30 +141,18 @@ public PositionOutputStream newOutputStream(String name) 5, null, executor)) { - for (List invalid : - Arrays.asList( - Arrays.asList(BinaryString.fromString("category-a"), 7), - Arrays.asList( - BinaryString.fromString("category-a"), - 7, - BinaryString.fromString("tag"), - BinaryString.fromString("extra")), - Arrays.asList(null, 7, BinaryString.fromString("tag"), null))) { - assertThatThrownBy(() -> reader.visitCompositeEqual(invalid)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Expected 3 composite key fields"); - } assertThat( - reader.visitCompositeEqual( - Arrays.asList(null, 7, BinaryString.fromString("tag"))) + reader.visitComposite( + equal(type, null, 7, BinaryString.fromString("tag"))) .get() .get() .results() .toRangeList()) .isEmpty(); assertThat( - reader.visitCompositeEqual( - Arrays.asList( + reader.visitComposite( + equal( + type, BinaryString.fromString("category-a"), -1, BinaryString.fromString(""))) @@ -154,8 +162,9 @@ public PositionOutputStream newOutputStream(String name) .toRangeList()) .containsExactly(new Range(0, 0)); assertThat( - reader.visitCompositeEqual( - Arrays.asList( + reader.visitComposite( + equal( + type, BinaryString.fromString("category-a"), 7, BinaryString.fromString("tag"))) @@ -165,8 +174,9 @@ public PositionOutputStream newOutputStream(String name) .toRangeList()) .containsExactly(new Range(2, 3)); assertThat( - reader.visitCompositeEqual( - Arrays.asList( + reader.visitComposite( + equal( + type, BinaryString.fromString("category-a"), 8, BinaryString.fromString("tag"))) @@ -175,6 +185,57 @@ public PositionOutputStream newOutputStream(String name) .results() .isEmpty()) .isTrue(); + PredicateBuilder builder = new PredicateBuilder(type); + Predicate fullKey = + equal( + type, + BinaryString.fromString("category-a"), + 7, + BinaryString.fromString("tag")); + for (Predicate unsupported : + Arrays.asList( + builder.equal(0, BinaryString.fromString("category-a")), + PredicateBuilder.and( + builder.equal(0, BinaryString.fromString("category-a")), + builder.greaterThan(1, 7), + builder.equal(2, BinaryString.fromString("tag"))), + PredicateBuilder.and( + builder.in( + 0, + Arrays.asList( + BinaryString.fromString("category-a"), + BinaryString.fromString("category-b"))), + builder.equal(1, 7), + builder.equal(2, BinaryString.fromString("tag"))), + PredicateBuilder.or(fullKey, builder.equal(1, 8)))) { + assertThat(reader.visitComposite(unsupported).join()).isEmpty(); + } + assertThat( + reader.visitComposite( + PredicateBuilder.and(fullKey, builder.equal(1, 8))) + .join() + .get() + .results() + .isEmpty()) + .isTrue(); + RowType predicateType = + new RowType( + Arrays.asList( + type.getFields().get(2), + type.getFields().get(0), + type.getFields().get(1))); + assertThat( + reader.visitComposite( + equal( + predicateType, + BinaryString.fromString("tag"), + BinaryString.fromString("category-a"), + 7)) + .join() + .get() + .results() + .toRangeList()) + .containsExactly(new Range(2, 3)); } try (GlobalIndexReader reader = indexer.createReader( @@ -184,8 +245,9 @@ public PositionOutputStream newOutputStream(String name) Collections.singletonList(new Range(3, 3)), executor)) { assertThat( - reader.visitCompositeEqual( - Arrays.asList( + reader.visitComposite( + equal( + type, BinaryString.fromString("category-a"), 7, BinaryString.fromString("tag"))) @@ -199,6 +261,24 @@ public PositionOutputStream newOutputStream(String name) } } + @Test + void testScalarReaderDoesNotSupportCompositePredicates() throws Exception { + RowType type = RowType.of(DataTypes.INT()); + GlobalIndexer indexer = GlobalIndexer.create("btree", type.getFields(), new Options()); + ExecutorService executor = newDirectExecutorService(); + try (GlobalIndexReader reader = + indexer.createReader( + meta -> LocalFileIO.create().newInputStream(meta.filePath()), + Collections.emptyList(), + 5, + null, + executor)) { + assertThat(reader.visitComposite(equal(type, 7)).join()).isEmpty(); + } finally { + executor.shutdownNow(); + } + } + @Test void testCompositeKeysPreserveTypesBoundariesAndNulls() { RowType type = @@ -213,18 +293,6 @@ void testCompositeKeysPreserveTypesBoundariesAndNulls() { KeySerializer serializer = new CompositeKeySerializer((RowType) indexer.keyExtractor().keyType()); Comparator comparator = serializer.createComparator(); - for (GenericRow invalid : - Arrays.asList( - GenericRow.of(BinaryString.fromString("category-a"), 7), - GenericRow.of( - BinaryString.fromString("category-a"), - 7, - BinaryString.fromString("tag"), - BinaryString.fromString("extra")))) { - assertThatThrownBy(() -> serializer.serialize(invalid)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Expected 3 composite key fields"); - } GenericRow first = row("category-a", -1, "a\u0000b"); GenericRow second = row("category-a", 107, ""); GenericRow nullable = row(null, 107, null); @@ -247,6 +315,107 @@ void testCompositeKeysPreserveTypesBoundariesAndNulls() { assertThat(indexer.keyExtractor().keyType()).isEqualTo(type); } + @Test + void testDeserializedKeysRemainIndependentOfSubsequentReadsAndInputBuffers() { + KeySerializer serializer = + new CompositeKeySerializer( + RowType.of(DataTypes.STRING(), DataTypes.INT(), DataTypes.STRING())); + GenericRow expectedFirst = row("category-a", 7, "first"); + GenericRow expectedSecond = row("category-b", 8, "second"); + byte[] firstBytes = serializer.serialize(expectedFirst); + Object first = serializer.deserialize(MemorySlice.wrap(firstBytes)); + Arrays.fill(firstBytes, (byte) 0); + byte[] secondBytes = serializer.serialize(expectedSecond); + Object second = serializer.deserialize(MemorySlice.wrap(secondBytes)); + Arrays.fill(secondBytes, (byte) 0); + for (int i = 0; i < 100; i++) { + serializer.deserialize(MemorySlice.wrap(serializer.serialize(row("other", i, null)))); + } + assertThat(first).isEqualTo(expectedFirst); + assertThat(second).isEqualTo(expectedSecond); + assertThat(serializer.createComparator().compare(first, second)).isNegative(); + } + + @Test + void testCompactedEncodingIgnoresRowKind() { + RowType type = RowType.of(DataTypes.STRING(), DataTypes.INT(), DataTypes.STRING()); + KeySerializer serializer = new CompositeKeySerializer(type); + GenericRow expected = row("category-a", -1, null); + byte[] compacted = new RowCompactedSerializer(type).serializeToBytes(expected); + for (RowKind kind : RowKind.values()) { + GenericRow key = row("category-a", -1, null); + key.setRowKind(kind); + byte[] bytes = serializer.serialize(key); + assertThat(bytes).isEqualTo(compacted); + assertThat(key.getRowKind()).isEqualTo(kind); + assertThat(serializer.deserialize(MemorySlice.wrap(bytes))).isEqualTo(expected); + assertThat(serializer.createComparator().compare(key, expected)).isZero(); + } + } + + @Test + void testEqualNaNKeysHaveIdenticalEncodings() { + KeySerializer serializer = + new CompositeKeySerializer(RowType.of(DataTypes.FLOAT(), DataTypes.DOUBLE())); + GenericRow canonical = GenericRow.of(Float.NaN, Double.NaN); + GenericRow alternate = + GenericRow.of( + Float.intBitsToFloat(0xffc00001), + Double.longBitsToDouble(0xfff8000000000001L)); + assertThat(serializer.createComparator().compare(canonical, alternate)).isZero(); + assertThat(serializer.serialize(alternate)).isEqualTo(serializer.serialize(canonical)); + assertThat(serializer.deserialize(MemorySlice.wrap(serializer.serialize(alternate)))) + .isEqualTo(canonical); + assertThat(Float.floatToRawIntBits(alternate.getFloat(0))).isEqualTo(0xffc00001); + assertThat(Double.doubleToRawLongBits(alternate.getDouble(1))) + .isEqualTo(0xfff8000000000001L); + } + + @Test + void testConcurrentRoundTrips() throws Exception { + RowType type = RowType.of(DataTypes.STRING(), DataTypes.INT(), DataTypes.STRING()); + KeySerializer serializer = new CompositeKeySerializer(type); + ExecutorService executor = Executors.newFixedThreadPool(8); + CountDownLatch start = new CountDownLatch(1); + List> tasks = new ArrayList<>(); + try { + for (int thread = 0; thread < 8; thread++) { + final int worker = thread; + tasks.add( + executor.submit( + () -> { + start.await(); + for (int i = 0; i < 500; i++) { + char[] chars = new char[64 + (i % 32) * 128]; + Arrays.fill(chars, (char) ('a' + worker)); + GenericRow key = + row( + "category-" + worker, + i, + i % 2 == 0 ? new String(chars) : null); + byte[] bytes = serializer.serialize(key); + Object restored = + serializer.deserialize(MemorySlice.wrap(bytes)); + assertThat(restored).isEqualTo(key); + assertThat( + serializer + .createComparator() + .compare(restored, key)) + .isZero(); + } + return null; + })); + } + start.countDown(); + for (Future task : tasks) { + task.get(30, TimeUnit.SECONDS); + } + } finally { + start.countDown(); + executor.shutdownNow(); + } + } + @Test void testFactoryRejectsUnsupportedFieldLists() { assertThatThrownBy( @@ -278,6 +447,15 @@ void testCompositeSupportDoesNotEnablePhysicalRowIndexes() { } } + private Predicate equal(RowType type, Object... values) { + PredicateBuilder builder = new PredicateBuilder(type); + List equalities = new ArrayList<>(); + for (int i = values.length - 1; i >= 0; i--) { + equalities.add(builder.equal(i, values[i])); + } + return PredicateBuilder.and(equalities); + } + private GenericRow row(String category, int itemNumber, String tag) { return GenericRow.of( category == null ? null : BinaryString.fromString(category), diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/CompositeBTreePredicateTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/CompositeBTreePredicateTest.java new file mode 100644 index 000000000000..6172018d29b7 --- /dev/null +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/CompositeBTreePredicateTest.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.globalindex.btree; + +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.fs.Path; +import org.apache.paimon.globalindex.CompositeKeySerializer; +import org.apache.paimon.globalindex.GlobalIndexIOMeta; +import org.apache.paimon.globalindex.KeySerializer; +import org.apache.paimon.globalindex.SortedIndexFileMeta; +import org.apache.paimon.predicate.LeafPredicate; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.PredicateBuilder; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowType; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Complete tuple matching and conservative manifest metadata pruning. */ +class CompositeBTreePredicateTest { + + private final List fields = + Arrays.asList( + new DataField(10, "category", DataTypes.STRING()), + new DataField(20, "item_number", DataTypes.INT())); + private final PredicateBuilder builder = new PredicateBuilder(new RowType(fields)); + private final BTreeGlobalIndexerFactory factory = new BTreeGlobalIndexerFactory(); + private final KeySerializer serializer = new CompositeKeySerializer(new RowType(fields)); + + @Test + void testMatchesInIndexOrderRegardlessOfPredicateOrder() { + Predicate predicate = + PredicateBuilder.and( + builder.equal(1, -3), builder.equal(0, BinaryString.fromString("a"))); + List matched = CompositeBTreePredicate.match(fields, predicate).get(); + assertThat( + matched.stream() + .map(leaf -> leaf.literals().get(0)) + .collect(Collectors.toList())) + .containsExactly(BinaryString.fromString("a"), -3); + assertThat(CompositeBTreePredicate.isContradictory(fields, predicate)).isFalse(); + assertThat( + CompositeBTreePredicate.match( + Arrays.asList(fields.get(1), fields.get(0)), predicate) + .get().stream() + .map(leaf -> leaf.literals().get(0)) + .collect(Collectors.toList())) + .containsExactly(-3, BinaryString.fromString("a")); + } + + @Test + void testRequiresCompleteConjunctiveEqualitiesAndMatchingTypes() { + assertThat( + CompositeBTreePredicate.match( + fields, builder.equal(0, BinaryString.fromString("a")))) + .isEmpty(); + assertThat( + CompositeBTreePredicate.match( + fields, + PredicateBuilder.or( + builder.equal(0, BinaryString.fromString("a")), + builder.equal(1, 7)))) + .isEmpty(); + assertThat( + CompositeBTreePredicate.match( + fields, + PredicateBuilder.and( + builder.equal(0, BinaryString.fromString("a")), + builder.greaterThan(1, 7)))) + .isEmpty(); + PredicateBuilder incompatible = + new PredicateBuilder( + new RowType( + Arrays.asList( + new DataField(10, "category", DataTypes.INT()), + fields.get(1)))); + assertThat( + CompositeBTreePredicate.match( + fields, + PredicateBuilder.and( + incompatible.equal(0, 1), incompatible.equal(1, 7)))) + .isEmpty(); + } + + @Test + void testPrunesUsingTypedTupleEndpointsInclusively() { + GlobalIndexIOMeta first = file("first", "a", -3, "a", 7); + GlobalIndexIOMeta second = file("second", "a", 10, "b", 2); + GlobalIndexIOMeta third = file("third", "b", 3, "c", 4); + List files = Arrays.asList(first, second, third); + assertThat(factory.selectFiles(fields, equal("a", -3), files)).containsExactly(first); + assertThat(factory.selectFiles(fields, equal("a", 7), files)).containsExactly(first); + assertThat(factory.selectFiles(fields, equal("a", 10), files)).containsExactly(second); + assertThat(factory.selectFiles(fields, equal("b", 2), files)).containsExactly(second); + assertThat(factory.selectFiles(fields, equal("b", 3), files)).containsExactly(third); + assertThat(factory.selectFiles(fields, equal("a", 8), files)).isEmpty(); + assertThat(factory.selectFiles(fields, equal("z", 7), files)).isEmpty(); + } + + @Test + void testContradictionsAndSqlNullEqualityPruneAllFiles() { + List files = Collections.singletonList(file("first", "a", -3, "a", 7)); + for (Predicate predicate : + Arrays.asList( + PredicateBuilder.and(equal("a", 7), builder.equal(1, 8)), + PredicateBuilder.and( + equal("a", 7), builder.equal(0, BinaryString.fromString("b"))), + PredicateBuilder.and(builder.equal(0, null), builder.equal(1, 7)), + PredicateBuilder.and( + builder.equal(0, BinaryString.fromString("a")), + builder.equal(1, null)))) { + assertThat(CompositeBTreePredicate.isContradictory(fields, predicate)).isTrue(); + assertThat(factory.selectFiles(fields, predicate, files)).isEmpty(); + } + Predicate redundant = PredicateBuilder.and(equal("a", 7), builder.equal(1, 7)); + assertThat(CompositeBTreePredicate.isContradictory(fields, redundant)).isFalse(); + assertThat(factory.selectFiles(fields, redundant, files)).containsExactlyElementsOf(files); + } + + @Test + void testRetainsFilesWhenCompleteKeyOrMetadataIsUnavailable() { + GlobalIndexIOMeta file = file("first", "a", -3, "a", 7); + List files = Collections.singletonList(file); + assertThat(factory.selectFiles(fields, builder.equal(1, 100), files)).isSameAs(files); + assertThat( + factory.selectFiles( + fields, + PredicateBuilder.and( + builder.equal(0, BinaryString.fromString("a")), + builder.greaterThan(1, 100)), + files)) + .isSameAs(files); + List unknown = + Arrays.asList(file, new GlobalIndexIOMeta(new Path("unknown"), 1, null)); + assertThat(factory.selectFiles(fields, equal("z", 7), unknown)).isSameAs(unknown); + assertThat( + factory.selectFiles( + fields, + PredicateBuilder.and(equal("a", 7), builder.equal(1, 8)), + unknown)) + .isEmpty(); + } + + private Predicate equal(String category, int number) { + return PredicateBuilder.and( + builder.equal(1, number), builder.equal(0, BinaryString.fromString(category))); + } + + private GlobalIndexIOMeta file( + String name, String first, int firstNumber, String last, int lastNumber) { + byte[] metadata = + new SortedIndexFileMeta( + serializer.serialize( + GenericRow.of(BinaryString.fromString(first), firstNumber)), + serializer.serialize( + GenericRow.of(BinaryString.fromString(last), lastNumber)), + false) + .serialize(); + return new GlobalIndexIOMeta(new Path(name), 1, metadata); + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java index 9ffcb69dd0c4..a1cdc71ebe38 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionBatchScan.java @@ -332,7 +332,7 @@ private Plan planEager() { return wrapToIndexSplits(splits, rowRangeIndex, scoreGetter); } - /** Plan complete data splits while deferring supported scalar index evaluation to readers. */ + /** Plan complete data splits while deferring index evaluation to readers. */ private Plan planWithIndexQuery() { if (queryAuthEnabled() || filter == null @@ -390,36 +390,25 @@ private Plan planWithIndexQuery() { indexFiles, table.store().pathFactory().globalIndexFileFactory()); if (indexQuery == null) { - return planEagerIndex(dataPlan, snapshot, partitionFilter, indexFiles, indexFilter); + return dataPlan; } - List unindexed = - new DataEvolutionGlobalIndexCoverage( - table, - snapshot, - partitionFilter, - indexFiles, - table.coreOptions().scalarIndexSearchMode()) - .unindexedRanges( - indexQuery.contributingFieldIds(table.rowType()), - table.coreOptions().scalarIndexSearchMode() - == CoreOptions.GlobalIndexSearchMode.DETAIL - ? GlobalIndexBuilderUtils.calcRowRanges( - splits.stream() - .map(DataEvolutionBatchScan::dataSplit) - .collect(Collectors.toList())) - : Collections.emptyList()); List indexQuerySplits = new ArrayList<>(); for (Split split : splits) { DataSplit dataSplit = dataSplit(split); List ranges = GlobalIndexBuilderUtils.calcRowRanges(Collections.singletonList(dataSplit)); GlobalIndexQuery splitQuery = indexQuery.forRanges(ranges); - List splitUnindexed = Range.and(unindexed, ranges); - if (splitQuery.isEmpty() && splitUnindexed.isEmpty()) { + List queryRanges = + table.coreOptions().scalarIndexSearchMode() + == CoreOptions.GlobalIndexSearchMode.FAST + ? Range.and(ranges, splitQuery.indexedRanges()) + : ranges; + if (queryRanges.isEmpty() || splitQuery.isEmpty(queryRanges)) { continue; } Split indexQuerySplit = - new IndexQuerySplit(dataSplit, splitQuery, table.options(), splitUnindexed); + new IndexQuerySplit( + dataSplit, splitQuery, table.options(), Collections.emptyList()); indexQuerySplits.add(withAuth(split, indexQuerySplit)); } return () -> indexQuerySplits; @@ -457,46 +446,6 @@ private static boolean supportsIndexQuery(CoreOptions options) { } } - private Plan planEagerIndex( - Plan dataPlan, - Snapshot snapshot, - PartitionPredicate partitionFilter, - List indexFiles, - Predicate indexFilter) { - try (DataEvolutionGlobalIndexScanner scanner = - DataEvolutionGlobalIndexScanner.create(table, snapshot, partitionFilter, indexFiles) - .get()) { - Optional result = - scanner.scanWithCoverage(indexFilter); - if (!result.isPresent()) { - return dataPlan; - } - GlobalIndexResult candidates = - result.get() - .result() - .or( - scanner.unindexedRowsForContributingFields( - result.get().contributingFieldIds())); - RowRangeIndex rowRangeIndex = RowRangeIndex.create(candidates.results().toRangeList()); - ScoreGetter scores = - candidates instanceof ScoredGlobalIndexResult - ? ((ScoredGlobalIndexResult) candidates).scoreGetter() - : null; - List indexed = new ArrayList<>(); - for (Split split : dataPlan.splits()) { - DataSplit dataSplit = dataSplit(split); - if (dataSplit.dataFiles().stream() - .map(DataFileMeta::nonNullRowIdRange) - .anyMatch(range -> rowRangeIndex.intersects(range.from, range.to))) { - indexed.add(withAuth(split, wrap(dataSplit, rowRangeIndex, scores))); - } - } - return () -> indexed; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - private static DataSplit dataSplit(Split split) { return (DataSplit) (split instanceof QueryAuthSplit ? ((QueryAuthSplit) split).split() : split); @@ -553,11 +502,7 @@ private Optional evalGlobalIndex() { if (result.isPresent()) { long coverageStart = System.nanoTime(); GlobalIndexResult finalResult = - result.get() - .result() - .or( - scanner.unindexedRowsForContributingFields( - result.get().contributingFieldIds())); + result.get().result().or(scanner.unindexedRowsForEvaluation(result.get())); long coverageDuration = System.nanoTime() - coverageStart; long totalDuration = System.nanoTime() - totalStart; LOG.info( diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexCoverage.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexCoverage.java index 6a945c054474..6ff33b520203 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexCoverage.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexCoverage.java @@ -67,13 +67,16 @@ public DataEvolutionGlobalIndexCoverage( this.coverageByField = new HashMap<>(); for (IndexFileMeta indexFile : indexFiles) { GlobalIndexMeta meta = checkNotNull(indexFile.globalIndexMeta()); + if (!DataEvolutionGlobalIndexScanner.isIndexInSchema(table.rowType(), indexFile)) { + continue; + } + // Single-field coverage belongs to single-field index definitions. Multi-field + // query paths supply their actual coverage explicitly. + if (meta.extraFieldIds() != null && meta.extraFieldIds().length > 0) { + continue; + } Range range = new Range(meta.rowRangeStart(), meta.rowRangeEnd()); addCoverage(meta.indexFieldId(), range); - if (meta.extraFieldIds() != null) { - for (int extra : meta.extraFieldIds()) { - addCoverage(extra, range); - } - } } } @@ -87,6 +90,11 @@ public List unindexedRanges(int fieldId) { public List unindexedRanges( Collection fieldIds, @Nullable List plannedDataRanges) { + return unindexedRangesFromCoverage(indexedRanges(fieldIds), plannedDataRanges); + } + + public List unindexedRangesFromCoverage( + List indexedRanges, @Nullable List plannedDataRanges) { if (searchMode == GlobalIndexSearchMode.FAST) { return Collections.emptyList(); } @@ -101,8 +109,7 @@ public List unindexedRanges( dataRanges = Collections.singletonList(new Range(0, snapshot.nextRowId() - 1)); } - List predicateIndexedRanges = - Range.sortAndMergeOverlap(indexedRanges(fieldIds), true); + List predicateIndexedRanges = Range.sortAndMergeOverlap(indexedRanges, true); List unindexedRanges = new ArrayList<>(); for (Range dataRange : Range.sortAndMergeOverlap(dataRanges, true)) { unindexedRanges.addAll(dataRange.exclude(predicateIndexedRanges)); diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexScanner.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexScanner.java index 4e65b3be0e7f..a6ef515ceb29 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexScanner.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/DataEvolutionGlobalIndexScanner.java @@ -50,6 +50,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -62,7 +63,6 @@ import static org.apache.paimon.CoreOptions.GLOBAL_INDEX_THREAD_NUM; import static org.apache.paimon.predicate.PredicateVisitor.collectFieldIds; import static org.apache.paimon.table.source.snapshot.TimeTravelUtil.tryTravelOrLatest; -import static org.apache.paimon.utils.Preconditions.checkArgument; import static org.apache.paimon.utils.Preconditions.checkNotNull; /** Scanner for shard-based global indexes on data-evolution tables. */ @@ -80,6 +80,8 @@ public class DataEvolutionGlobalIndexScanner implements Closeable { private final IndexPathFactory indexPathFactory; private final DataEvolutionGlobalIndexCoverage coverage; private final FileStoreTable table; + private final List indexFiles; + private final FileIO fileIO; private DataEvolutionGlobalIndexScanner( FileStoreTable table, @@ -113,6 +115,8 @@ private DataEvolutionGlobalIndexScanner( Collection coverageIndexFiles, Collection indexFiles) { this.table = table; + this.indexFiles = new ArrayList<>(indexFiles); + this.fileIO = fileIO; this.options = options; this.rowType = rowType; this.executor = @@ -147,23 +151,15 @@ private DataEvolutionGlobalIndexScanner( /** Groups metadata for both planning-time readers and reader-side query plans. */ static Map> groupIndexFiles( Collection indexFiles) { - Map primaryGroups = new HashMap<>(); + Map, IndexMetaFileGroup> primaryGroups = new LinkedHashMap<>(); for (IndexFileMeta indexFile : indexFiles) { GlobalIndexMeta meta = checkNotNull(indexFile.globalIndexMeta()); int indexFieldId = meta.indexFieldId(); List fieldIds = meta.getIndexedFieldIds(); - IndexMetaFileGroup group = primaryGroups.get(indexFieldId); + IndexMetaFileGroup group = primaryGroups.get(fieldIds); if (group == null) { group = new IndexMetaFileGroup(indexFieldId, fieldIds); - primaryGroups.put(indexFieldId, group); - } else { - checkArgument( - group.fieldIds.equals(fieldIds), - "Primary field %s owns multiple indexes with different columns %s and %s; " - + "a primary column can own at most one index.", - indexFieldId, - group.fieldIds, - fieldIds); + primaryGroups.put(fieldIds, group); } group.addFile(indexFile.indexType(), meta.rowRange(), indexFile); } @@ -224,12 +220,30 @@ public static Optional create( return create(table, null, null, indexFiles); } + /** Search pre-filters use per-column coverage, so they require single-field definitions. */ + public static Optional createForScalarFilters( + FileStoreTable table, + @Nullable Snapshot pinnedSnapshot, + @Nullable PartitionPredicate partitionFilter, + Collection indexFiles) { + return create( + table, + pinnedSnapshot, + partitionFilter, + indexFiles.stream() + .filter(file -> !isMultiFieldIndex(file)) + .collect(Collectors.toList())); + } + public static Optional create( FileStoreTable table, @Nullable Snapshot pinnedSnapshot, @Nullable PartitionPredicate partitionFilter, Collection indexFiles) { - List globalIndexFiles = globalIndexFiles(indexFiles); + List globalIndexFiles = + globalIndexFiles(indexFiles).stream() + .filter(file -> isIndexInSchema(table.rowType(), file)) + .collect(Collectors.toList()); if (globalIndexFiles.isEmpty()) { return Optional.empty(); } @@ -325,6 +339,8 @@ private static Filter topNIndexFileFilter( GlobalIndexMeta globalIndex = indexFile.globalIndexMeta(); return globalIndex != null && globalIndex.indexFieldId() == fieldId + && (globalIndex.extraFieldIds() == null + || globalIndex.extraFieldIds().length == 0) && BTreeGlobalIndexerFactory.IDENTIFIER.equals(indexFile.indexType()); }; } @@ -343,11 +359,12 @@ static Filter indexFileFilter( return false; } GlobalIndexMeta globalIndex = entry.indexFile().globalIndexMeta(); - if (globalIndex == null) { + if (globalIndex == null + || !isIndexInSchema(table.rowType(), entry.indexFile())) { return false; } // Collect indexes whose primary column is filtered, and also multi-column - // indexes that have a filtered column as an extra (used as a fallback). + // indexes that have a filtered column as an extra. if (filterFieldIds.contains(globalIndex.indexFieldId())) { return true; } @@ -363,6 +380,17 @@ static Filter indexFileFilter( return indexFileFilter; } + private static boolean isMultiFieldIndex(IndexFileMeta file) { + GlobalIndexMeta meta = file.globalIndexMeta(); + return meta != null && meta.extraFieldIds() != null && meta.extraFieldIds().length > 0; + } + + /** Dropped indexed columns leave manifest entries that cannot serve the current schema. */ + static boolean isIndexInSchema(RowType rowType, IndexFileMeta file) { + return file.globalIndexMeta().getIndexedFieldIds().stream() + .allMatch(rowType::containsField); + } + private static List globalIndexFiles(Collection indexFiles) { return indexFiles.stream() .filter(indexFile -> indexFile.globalIndexMeta() != null) @@ -370,10 +398,29 @@ private static List globalIndexFiles(Collection in } public Optional scan(Predicate predicate) { - return globalIndexEvaluator.evaluate(predicate); + return scanWithCoverage(predicate).map(GlobalIndexEvaluator.Evaluation::result); } public Optional scanWithCoverage(Predicate predicate) { + GlobalIndexQuery query = + predicate == null + || indexFiles.stream() + .noneMatch( + DataEvolutionGlobalIndexScanner::isMultiFieldIndex) + ? null + : GlobalIndexQuery.create(rowType, predicate, indexFiles, indexPathFactory); + if (query != null && query.hasCompositeQuery()) { + try { + return query.evaluateWithCoverage( + fileIO, + options, + indexFiles.stream() + .map(file -> file.globalIndexMeta().rowRange()) + .collect(Collectors.toList())); + } catch (IOException e) { + throw new RuntimeException("Failed to evaluate composite global index query", e); + } + } return globalIndexEvaluator.evaluateWithContributingFields(predicate); } @@ -393,12 +440,38 @@ public Optional scan(TopN topN) { public GlobalIndexResult unindexedRows(Predicate predicate) { RoaringNavigableMap64 rows = new RoaringNavigableMap64(); - for (Range range : coverage.unindexedRanges(rowType, predicate)) { + GlobalIndexQuery query = + predicate == null + ? null + : GlobalIndexQuery.create(rowType, predicate, indexFiles, indexPathFactory); + if (query != null && query.hasCompositeQuery() && query.hasScalarQuery()) { + // Scalar shards can decline a predicate at runtime, so the legacy API must use + // evaluated coverage as well. Internal callers retain the evaluation directly. + Optional evaluation = scanWithCoverage(predicate); + return evaluation.isPresent() + ? unindexedRowsForEvaluation(evaluation.get()) + : GlobalIndexResult.fromRanges( + coverage.unindexedRangesFromCoverage(Collections.emptyList(), null)); + } + List unindexed = + query != null && query.hasCompositeQuery() + ? coverage.unindexedRangesFromCoverage(query.coveredRanges(), null) + : coverage.unindexedRanges(rowType, predicate); + for (Range range : unindexed) { rows.addRange(range); } return GlobalIndexResult.create(rows); } + public GlobalIndexResult unindexedRowsForEvaluation( + GlobalIndexEvaluator.Evaluation evaluation) { + if (evaluation.coveredRanges() == null) { + return unindexedRowsForContributingFields(evaluation.contributingFieldIds()); + } + return GlobalIndexResult.fromRanges( + coverage.unindexedRangesFromCoverage(evaluation.coveredRanges(), null)); + } + public GlobalIndexResult unindexedRowsForContributingFields( Collection contributingFieldIds) { RoaringNavigableMap64 rows = new RoaringNavigableMap64(); @@ -421,6 +494,9 @@ private Collection createReaders( GlobalIndexFileReader indexFileReadWrite, IndexMetaFileGroup group, RowType rowType) { List indexFields = group.fieldIds.stream().map(rowType::getField).collect(Collectors.toList()); + if (indexFields.size() > 1) { + return Collections.emptyList(); + } Set readers = new HashSet<>(); for (Map.Entry>> entry : group.metas.entrySet()) { diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexQuery.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexQuery.java index 7446ce5ed1a1..18737d69f98e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexQuery.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/GlobalIndexQuery.java @@ -21,6 +21,8 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.globalindex.DataEvolutionGlobalIndexScanner.IndexMetaFileGroup; +import org.apache.paimon.globalindex.GlobalIndexEvaluator.Evaluation; +import org.apache.paimon.globalindex.btree.CompositeBTreePredicate; import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.index.IndexPathFactory; import org.apache.paimon.io.DataInputView; @@ -28,12 +30,14 @@ import org.apache.paimon.options.Options; import org.apache.paimon.predicate.And; import org.apache.paimon.predicate.CompoundPredicate; +import org.apache.paimon.predicate.Equal; import org.apache.paimon.predicate.FieldRef; import org.apache.paimon.predicate.GreaterOrEqual; import org.apache.paimon.predicate.LeafPredicate; import org.apache.paimon.predicate.LessOrEqual; import org.apache.paimon.predicate.Or; import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.types.DataField; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.InstantiationUtil; @@ -58,6 +62,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.function.Function; +import java.util.stream.Collectors; import static org.apache.paimon.CoreOptions.GLOBAL_INDEX_THREAD_NUM; import static org.apache.paimon.predicate.PredicateVisitor.collectFieldIds; @@ -73,7 +78,7 @@ */ class GlobalIndexQuery { - /** A leaf or paired range query; null for an AND/OR node. */ + /** A leaf, paired range or composite lookup; null for an AND/OR node. */ @Nullable private final Predicate predicate; /** For compound nodes, true means OR and false means AND; ignored for leaves. */ @@ -103,7 +108,13 @@ static GlobalIndexQuery create( List files, IndexPathFactory pathFactory) { Map> groupsByField = - DataEvolutionGlobalIndexScanner.groupIndexFiles(files); + DataEvolutionGlobalIndexScanner.groupIndexFiles( + files.stream() + .filter( + file -> + DataEvolutionGlobalIndexScanner.isIndexInSchema( + rowType, file)) + .collect(Collectors.toList())); Map> groups = new LinkedHashMap<>(); groupsByField.forEach( (fieldId, fieldGroups) -> { @@ -131,6 +142,72 @@ private static GlobalIndexQuery createForPredicate( boolean union = compound.function() instanceof Or; List children = new ArrayList<>(); List predicates = GlobalIndexEvaluator.normalizedChildren(compound); + if (!union) { + // Prefer the longest full composite key before any single-column posting is read. + while (!predicates.isEmpty()) { + IndexGroup selected = null; + List matched = null; + Predicate conjunction = PredicateBuilder.and(predicates); + for (List fieldGroups : groups.values()) { + for (IndexGroup group : fieldGroups) { + if (!group.isCompositeBTree() + || (selected != null + && group.extraFields.size() + <= selected.extraFields.size())) { + continue; + } + Optional> match = + CompositeBTreePredicate.match(group.indexFields(), conjunction); + if (match.isPresent()) { + selected = group; + matched = match.get(); + } + } + } + if (selected == null) { + break; + } + List covered = new ArrayList<>(); + for (Predicate child : predicates) { + if (child instanceof LeafPredicate) { + LeafPredicate leaf = (LeafPredicate) child; + if (leaf.function() instanceof Equal + && matched.stream() + .anyMatch( + equality -> + equality.fieldRefOptional() + .equals(leaf.fieldRefOptional()))) { + covered.add(child); + } + } + } + Predicate composite = PredicateBuilder.and(covered); + List selectedGroups = new ArrayList<>(); + for (IndexGroup group : groups.get(selected.field.id())) { + if (group.type.equals(selected.type) + && group.indexFields().equals(selected.indexFields())) { + selectedGroups.add(group.selectFiles(composite)); + } + } + children.add( + new GlobalIndexQuery( + composite, false, Collections.emptyList(), selectedGroups)); + predicates.removeAll(covered); + // The tuple equality already narrows these columns to one value. Leave their + // remaining conditions as data filters instead of reading scalar postings. + List keyEqualities = matched; + predicates.removeIf( + child -> { + if (!(child instanceof LeafPredicate)) { + return false; + } + Optional field = ((LeafPredicate) child).fieldRefOptional(); + return keyEqualities.stream() + .anyMatch( + equality -> equality.fieldRefOptional().equals(field)); + }); + } + } for (int i = 0; i < predicates.size(); i++) { Predicate child = predicates.get(i); GlobalIndexQuery query = null; @@ -188,31 +265,64 @@ private static GlobalIndexQuery createForIndexedField( } List selectedGroups = new ArrayList<>(); for (IndexGroup group : fieldGroups) { - List selectedFiles = - GlobalIndexerFactoryUtils.selectFiles( - group.type, group.indexFields(), predicate, group.files); - if (!selectedFiles.isEmpty()) { - selectedGroups.add( - selectedFiles == group.files - ? group - : new IndexGroup( - group.type, - group.field, - group.extraFields, - group.range, - selectedFiles)); + // Scalar lookups use independent single-field index definitions. + if (group.extraFields.isEmpty()) { + selectedGroups.add(group.selectFiles(predicate)); } } + if (selectedGroups.isEmpty()) { + return null; + } return new GlobalIndexQuery(predicate, false, Collections.emptyList(), selectedGroups); } - boolean isEmpty() { + /** Only prune when metadata proves no match throughout the requested row ranges. */ + boolean isEmpty(List ranges) { if (predicate != null) { - return groups.isEmpty(); + return !groups.isEmpty() + && groups.stream().allMatch(group -> group.files.isEmpty()) + && Range.and(coveredRanges(), ranges).equals(ranges); } return union - ? children.stream().allMatch(GlobalIndexQuery::isEmpty) - : children.stream().anyMatch(GlobalIndexQuery::isEmpty); + ? children.stream().allMatch(child -> child.isEmpty(ranges)) + : children.stream().anyMatch(child -> child.isEmpty(ranges)); + } + + boolean hasCompositeQuery() { + return groups.stream().anyMatch(IndexGroup::isCompositeBTree) + || children.stream().anyMatch(GlobalIndexQuery::hasCompositeQuery); + } + + boolean hasScalarQuery() { + return groups.stream().anyMatch(group -> !group.isCompositeBTree()) + || children.stream().anyMatch(GlobalIndexQuery::hasScalarQuery); + } + + /** Rows covered by any selected index path, independent of predicate support. */ + List indexedRanges() { + return predicate != null + ? coveredRanges() + : Range.sortAndMergeOverlap( + children.stream() + .flatMap(child -> child.indexedRanges().stream()) + .collect(Collectors.toList()), + true); + } + + /** Coverage of the selected query paths, including files pruned safely by key metadata. */ + List coveredRanges() { + if (predicate != null) { + return Range.sortAndMergeOverlap( + groups.stream().map(group -> group.range).collect(Collectors.toList()), true); + } + List coverage = null; + for (GlobalIndexQuery child : children) { + coverage = + coverage == null + ? child.coveredRanges() + : Range.and(coverage, child.coveredRanges()); + } + return coverage == null ? Collections.emptyList() : coverage; } /** Residual predicates discarded during planning must not expand unindexed coverage. */ @@ -247,7 +357,7 @@ GlobalIndexQuery forRanges(List ranges) { for (GlobalIndexQuery child : children) { selectedChildren.add(child.forRanges(ranges)); } - // A query without a local group evaluates to an empty result. + // Keep children without local groups so readers can retain their uncertain candidates. return new GlobalIndexQuery(predicate, union, selectedChildren, selected); } @@ -255,28 +365,106 @@ GlobalIndexResult evaluate(FileIO fileIO, Options options, List ranges) throws IOException { ExecutorService executor = GlobalIndexReadThreadPool.getExecutorService(options.get(GLOBAL_INDEX_THREAD_NUM)); - return evaluateWithExecutor(fileIO, options, ranges, executor); + return evaluateWithExecutor(fileIO, options, ranges, executor, false).get().result(); + } + + /** Retain candidates wherever a child predicate cannot be evaluated by an index. */ + Optional evaluateCandidates( + FileIO fileIO, Options options, List ranges) throws IOException { + if (ranges.isEmpty()) { + return Optional.of(GlobalIndexResult.createEmpty()); + } + if (predicate == null) { + GlobalIndexResult result = null; + for (GlobalIndexQuery child : children) { + Optional candidates = + child.evaluateCandidates(fileIO, options, ranges); + if (!candidates.isPresent()) { + if (union) { + return Optional.empty(); + } + continue; + } + result = + result == null + ? candidates.get() + : union + ? result.or(candidates.get()) + : result.and(candidates.get()); + } + return Optional.ofNullable(result); + } + if (groups.isEmpty()) { + return Optional.empty(); + } + Optional evaluation = evaluateWithCoverage(fileIO, options, ranges); + return evaluation.map( + supported -> { + List uncovered = new ArrayList<>(); + for (Range range : ranges) { + uncovered.addAll(range.exclude(supported.coveredRanges())); + } + // Add gaps before AND/OR combination, so another index can still prune them. + return supported.result().or(GlobalIndexResult.fromRanges(uncovered)); + }); + } + + /** Preserve unsupported-reader fallback and coverage of the paths that actually contributed. */ + Optional evaluateWithCoverage(FileIO fileIO, Options options, List ranges) + throws IOException { + ExecutorService executor = + GlobalIndexReadThreadPool.getExecutorService(options.get(GLOBAL_INDEX_THREAD_NUM)); + return evaluateWithExecutor(fileIO, options, ranges, executor, true); } - private GlobalIndexResult evaluateWithExecutor( - FileIO fileIO, Options options, List ranges, ExecutorService executor) + private Optional evaluateWithExecutor( + FileIO fileIO, + Options options, + List ranges, + ExecutorService executor, + boolean allowUnsupported) throws IOException { if (predicate == null) { GlobalIndexResult result = null; + Set fields = new HashSet<>(); + List coverage = null; for (GlobalIndexQuery child : children) { - GlobalIndexResult matches = - child.evaluateWithExecutor(fileIO, options, ranges, executor); + Optional evaluation = + child.evaluateWithExecutor( + fileIO, options, ranges, executor, allowUnsupported); + if (!evaluation.isPresent()) { + if (union) { + return Optional.empty(); + } + continue; + } + GlobalIndexResult matches = evaluation.get().result(); result = result == null ? matches : union ? result.or(matches) : result.and(matches); + fields.addAll(evaluation.get().contributingFieldIds()); + List childCoverage = evaluation.get().coveredRanges(); + coverage = coverage == null ? childCoverage : Range.and(coverage, childCoverage); } - return result == null ? GlobalIndexResult.createEmpty() : result; + return result == null + ? Optional.empty() + : Optional.of(new Evaluation(result, fields, coverage)); } - Function>> query = - predicateQuery(); GlobalIndexResult result = GlobalIndexResult.createEmpty(); + Set fields = new HashSet<>(); + List coverage = new ArrayList<>(); + boolean supported = groups.isEmpty(); for (IndexGroup group : groups) { + if (group.files.isEmpty()) { + supported = true; + fields.addAll(collectFieldIds(new RowType(group.indexFields()), predicate)); + coverage.add(group.range); + continue; + } + Function>> query = + predicateQuery(group); GlobalIndexResult splitRows = localSplitRows(ranges, group.range); if (splitRows.results().isEmpty()) { + supported = true; continue; } GlobalIndexer indexer = @@ -290,8 +478,14 @@ private GlobalIndexResult evaluateWithExecutor( executor)) { Optional matches = query.apply(reader).get(); if (!matches.isPresent()) { + if (allowUnsupported) { + continue; + } throw new IOException("Index reader does not support predicate: " + predicate); } + supported = true; + fields.addAll(collectFieldIds(new RowType(group.indexFields()), predicate)); + coverage.add(group.range); // Clip in index-local coordinates before offset() iterates the retained rows. result = result.or(splitRows.and(matches.get()).offset(group.range.from)); } catch (InterruptedException e) { @@ -301,11 +495,17 @@ private GlobalIndexResult evaluateWithExecutor( throw new IOException("Failed to evaluate index query split", e.getCause()); } } - return result; + return supported + ? Optional.of( + new Evaluation(result, fields, Range.sortAndMergeOverlap(coverage, true))) + : Optional.empty(); } private Function>> - predicateQuery() { + predicateQuery(IndexGroup group) { + if (group.isCompositeBTree()) { + return reader -> reader.visitComposite(predicate); + } if (predicate instanceof LeafPredicate) { LeafPredicate leaf = (LeafPredicate) predicate; return reader -> @@ -445,8 +645,9 @@ public int hashCode() { } /** - * Files read together by one index reader, grouped by primary column, index type and row range. - * A group may overlap multiple data splits; primary and extra column predicates can share it. + * Files read together by one index reader, grouped by ordered index columns, index type and row + * range. A group may overlap multiple data splits; primary and extra column predicates can + * share it. */ private static class IndexGroup { /** Factory identifier used to create the index reader on the worker. */ @@ -512,6 +713,19 @@ private static List fromMetadata( return result; } + private boolean isCompositeBTree() { + return "btree".equals(type) && !extraFields.isEmpty(); + } + + private IndexGroup selectFiles(Predicate predicate) { + List selected = + GlobalIndexerFactoryUtils.selectFiles(type, indexFields(), predicate, files); + // Retain the row range even when metadata proves there are no matching files. + return selected == files + ? this + : new IndexGroup(type, field, extraFields, range, selected); + } + @Override public boolean equals(Object obj) { if (!(obj instanceof IndexGroup)) { diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/IndexQuerySplit.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/IndexQuerySplit.java index 11c3fc14cc56..51c998960689 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/IndexQuerySplit.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/IndexQuerySplit.java @@ -18,6 +18,7 @@ package org.apache.paimon.globalindex; +import org.apache.paimon.CoreOptions; import org.apache.paimon.fs.FileIO; import org.apache.paimon.io.DataInputView; import org.apache.paimon.io.DataInputViewStreamWrapper; @@ -37,6 +38,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.OptionalLong; /** @@ -89,9 +91,17 @@ public DataSplit dataSplit() { public IndexedSplit evaluate(FileIO fileIO) throws IOException { List ranges = GlobalIndexBuilderUtils.calcRowRanges(Collections.singletonList(dataSplit)); - GlobalIndexResult matches = - indexQuery.evaluate(fileIO, Options.fromMap(indexOptions), ranges); - List candidates = new ArrayList<>(matches.results().toRangeList()); + Options options = Options.fromMap(indexOptions); + if (new CoreOptions(options).scalarIndexSearchMode() + == CoreOptions.GlobalIndexSearchMode.FAST) { + ranges = Range.and(ranges, indexQuery.indexedRanges()); + } + Optional result = indexQuery.evaluateCandidates(fileIO, options, ranges); + if (!result.isPresent()) { + // No safe index query remains, including an OR with an unsupported branch. + return new IndexedSplit(dataSplit, ranges, null); + } + List candidates = new ArrayList<>(result.get().results().toRangeList()); candidates.addAll(unindexedRanges); return new IndexedSplit(dataSplit, Range.sortAndMergeOverlap(candidates, true), null); } diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexScanner.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexScanner.java index 7f44a334515b..753725793c03 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexScanner.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexScanner.java @@ -58,7 +58,7 @@ public class SortedGlobalIndexScanner implements Serializable { private final RowType rowType; private final Options options; - private DataField indexField; + private List indexFields; @Nullable private Snapshot snapshot; @@ -75,13 +75,17 @@ public SortedGlobalIndexScanner(Table table, String indexType, Options options) this.options = options; } - public SortedGlobalIndexScanner withIndexField(String indexField) { - checkArgument( - rowType.containsField(indexField), - "Column '%s' does not exist in table '%s'.", - indexField, - table.fullName()); - this.indexField = rowType.getField(indexField); + public SortedGlobalIndexScanner withIndexFields(List fieldNames) { + checkArgument(!fieldNames.isEmpty(), "At least one index column is required."); + this.indexFields = new ArrayList<>(); + for (String name : fieldNames) { + checkArgument( + rowType.containsField(name), + "Column '%s' does not exist in table '%s'.", + name, + table.fullName()); + indexFields.add(rowType.getField(name)); + } return this; } @@ -132,25 +136,16 @@ public Optional> incrementalScan() { } snapshotReader = withManifestEntryFilter(snapshotReader.withSnapshot(snapshot)); - Preconditions.checkArgument(indexField != null, "indexField must be set before scan."); + Preconditions.checkArgument(indexFields != null, "indexFields must be set before scan."); List currentIndexes = - currentIndexEntries( - table, - snapshot, - indexType, - Collections.singletonList(indexField), - partitionPredicate); + currentIndexEntries(table, snapshot, indexType, indexFields, partitionPredicate); List rangesToBuild = new ArrayList<>(unindexedRowRanges(snapshot, currentIndexes)); List deletedIndexEntries = Collections.emptyList(); if (detectDataFileChange()) { // Scans data manifests through reusable binary views without materializing entries. deletedIndexEntries = DataEvolutionGlobalIndexRefreshPlanner.findIndexesToRefresh( - table, - snapshot, - partitionPredicate, - currentIndexes, - Collections.singletonList(indexField)); + table, snapshot, partitionPredicate, currentIndexes, indexFields); for (IndexManifestEntry entry : deletedIndexEntries) { rangesToBuild.add(entry.indexFile().globalIndexMeta().rowRange()); } diff --git a/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java b/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java index 52f33840105a..9a5f62cb9797 100644 --- a/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriter.java @@ -43,7 +43,6 @@ import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; -import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -63,7 +62,7 @@ public class SortedGlobalIndexWriter implements Serializable { private final Options options; private final long recordsPerRange; - private DataField indexField; + private List indexFields; private GlobalIndexKeyExtractor keyExtractor; public SortedGlobalIndexWriter(Table table, String indexType) { @@ -79,16 +78,18 @@ public SortedGlobalIndexWriter(Table table, String indexType, Options options) { (long) (options.get(SortedIndexOptions.SORTED_INDEX_RECORDS_PER_FILE) * FLOATING); } - public SortedGlobalIndexWriter withIndexField(String indexField) { - checkArgument( - rowType.containsField(indexField), - "Column '%s' does not exist in table '%s'.", - indexField, - table.fullName()); - this.indexField = rowType.getField(indexField); - GlobalIndexer indexer = - GlobalIndexer.create( - indexType, Collections.singletonList(this.indexField), options); + public SortedGlobalIndexWriter withIndexFields(List fieldNames) { + checkArgument(!fieldNames.isEmpty(), "At least one index column is required."); + this.indexFields = new ArrayList<>(); + for (String name : fieldNames) { + checkArgument( + rowType.containsField(name), + "Column '%s' does not exist in table '%s'.", + name, + table.fullName()); + indexFields.add(rowType.getField(name)); + } + GlobalIndexer indexer = GlobalIndexer.create(indexType, indexFields, options); checkArgument( indexer instanceof SortedGlobalIndexer, "Index algorithm %s does not expose sorted index keys.", @@ -133,8 +134,7 @@ public SortedSingleColumnIndexWriter createTaskWriter(Range rowRange) throws IOE } public GlobalIndexSingleColumnWriter createWriter() throws IOException { - GlobalIndexWriter indexWriter = - createIndexWriter(table, indexType, Collections.singletonList(indexField), options); + GlobalIndexWriter indexWriter = createIndexWriter(table, indexType, indexFields, options); if (!(indexWriter instanceof GlobalIndexSingleColumnWriter)) { throw new RuntimeException( "Unexpected implementation, the index writer of " @@ -158,7 +158,7 @@ public CommitMessage flushIndex( table.store().pathFactory().globalIndexFileFactory(), table.coreOptions(), rowRange, - Collections.singletonList(indexField), + indexFields, indexType, resultEntries, sourceMeta); diff --git a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java index 16cfd163ed1c..c313a8ebcb6e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java +++ b/paimon-core/src/main/java/org/apache/paimon/manifest/IndexManifestFileHandler.java @@ -29,7 +29,6 @@ import javax.annotation.Nullable; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -243,6 +242,7 @@ private void validateRetainedIndexFiles( if (retainedMeta == null) { continue; } + List retainedFieldIds = retainedMeta.getIndexedFieldIds(); for (IndexManifestEntry added : addedIndexFiles) { GlobalIndexMeta addedMeta = added.indexFile().globalIndexMeta(); @@ -253,14 +253,12 @@ private void validateRetainedIndexFiles( retainedMeta.sourceMeta()) && !DataEvolutionIndexSourceMeta.isDataEvolutionMeta( addedMeta.sourceMeta())) - || retainedMeta.indexFieldId() != addedMeta.indexFieldId() - || (Arrays.equals( - retainedMeta.extraFieldIds(), addedMeta.extraFieldIds()) - && !Range.intersect( - retainedMeta.rowRangeStart(), - retainedMeta.rowRangeEnd(), - addedMeta.rowRangeStart(), - addedMeta.rowRangeEnd()))) { + || !Range.intersect( + retainedMeta.rowRangeStart(), + retainedMeta.rowRangeEnd(), + addedMeta.rowRangeStart(), + addedMeta.rowRangeEnd()) + || !retainedFieldIds.equals(addedMeta.getIndexedFieldIds())) { continue; } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java index c638a400a36a..5163bb580ef0 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataEvolutionVectorRead.java @@ -242,7 +242,7 @@ private RoaringNavigableMap64 scalarMatchedRows(List spl } Optional optionalScanner = - DataEvolutionGlobalIndexScanner.create( + DataEvolutionGlobalIndexScanner.createForScalarFilters( table, planSnapshot, partitionFilter, scalarIndexFiles); if (!optionalScanner.isPresent()) { return null; @@ -286,7 +286,7 @@ protected RoaringNavigableMap64 rawPreFilter(List splits) scalarIndexFiles.addAll(split.scalarIndexFiles()); } Optional optionalScanner = - DataEvolutionGlobalIndexScanner.create( + DataEvolutionGlobalIndexScanner.createForScalarFilters( table, planSnapshot, partitionFilter, scalarIndexFiles); if (!optionalScanner.isPresent()) { return null; diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java index 05d16d885268..a3bbc6d1a669 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataEvolutionFullTextRead.java @@ -260,7 +260,7 @@ private RoaringNavigableMap64 matchedRows( private Optional evaluateWithIndexes( Set scalarIndexFiles, @Nullable Snapshot planSnapshot) { Optional optionalScanner = - DataEvolutionGlobalIndexScanner.create( + DataEvolutionGlobalIndexScanner.createForScalarFilters( table, planSnapshot, partitionFilter, scalarIndexFiles); if (!optionalScanner.isPresent()) { return Optional.empty(); diff --git a/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java index e48bf52b7702..0cdcb9879106 100644 --- a/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java +++ b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java @@ -2372,7 +2372,8 @@ private Map anchorFilesByRange(FileStoreTable table) { private List buildSortedIndex( FileStoreTable table, String indexType, String indexFieldName) throws Exception { SortedGlobalIndexScanner scanner = - new SortedGlobalIndexScanner(table, indexType).withIndexField(indexFieldName); + new SortedGlobalIndexScanner(table, indexType) + .withIndexFields(Collections.singletonList(indexFieldName)); ScanResult scanResult = scanner.scan() .orElseThrow( diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java index 178577375fc3..13ece4e01979 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java @@ -3368,7 +3368,8 @@ private List dataManifestFileNames(FileStoreTable table) { private void createBTreeIndex(FileStoreTable table) throws Exception { SortedGlobalIndexScanner builder = - new SortedGlobalIndexScanner(table, "btree").withIndexField("id"); + new SortedGlobalIndexScanner(table, "btree") + .withIndexFields(Collections.singletonList("id")); ScanResult scanResult = builder.scan() .orElseThrow( diff --git a/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexQueryTest.java b/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexQueryTest.java index 7534931a31c0..79727b7b3e13 100644 --- a/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexQueryTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/globalindex/GlobalIndexQueryTest.java @@ -568,21 +568,14 @@ void testGroupsPreserveIndexTypesRangesAndExtraFields() { } @Test - void testGroupsRejectInconsistentIndexedFields() { - assertThatThrownBy( - () -> - DataEvolutionGlobalIndexScanner.groupIndexFiles( - Arrays.asList( - indexFile( - "es-index", - "first", - 0, - 99, - 10, - new int[] {20}), - indexFile("es-index", "tail", 100, 199, 10, null)))) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("different columns"); + void testGroupsAllowSingleAndCompositeIndexesWithTheSamePrimaryField() { + Map> groups = + DataEvolutionGlobalIndexScanner.groupIndexFiles( + Arrays.asList( + indexFile("btree", "composite", 0, 99, 10, new int[] {20}), + indexFile("btree", "single", 100, 199, 10, null))); + assertThat(groups.get(10)).hasSize(2); + assertThat(groups.get(20)).hasSize(1); } @Test @@ -603,10 +596,10 @@ void testPlansAvailableIndexesWithoutTypeChecks() throws Exception { null, new SortedIndexFileMeta(key, key, false).serialize()), null); - IndexFileMeta multiColumn = indexFile("custom-index", "multi", 0, 99, 1, new int[] {2}); + IndexFileMeta otherType = indexFile("custom-index", "single", 0, 99, 1, null); IndexPathFactory paths = mock(IndexPathFactory.class); when(paths.toPath(any(IndexFileMeta.class))).thenReturn(new Path("index")); - List files = Arrays.asList(supported, multiColumn); + List files = Arrays.asList(supported, otherType); Predicate supportedLeaf = builder.equal(0, 1); Predicate unsupportedLeaf = builder.equal(1, 2); @@ -638,6 +631,41 @@ void testPlansAvailableIndexesWithoutTypeChecks() throws Exception { .isNotNull(); } + @ParameterizedTest + @ValueSource(strings = {"btree", "custom-index"}) + void testScalarPlansUseOnlySingleFieldDefinitions(String indexType) { + RowType rowType = RowType.of(DataTypes.INT(), DataTypes.INT()); + PredicateBuilder builder = new PredicateBuilder(rowType); + IndexFileMeta multi = indexFile(indexType, "multi", 0, 99, 0, new int[] {1}); + IndexPathFactory paths = mock(IndexPathFactory.class); + when(paths.toPath(any(IndexFileMeta.class))) + .thenAnswer( + invocation -> + new Path(invocation.getArgument(0).fileName())); + for (int field : new int[] {0, 1}) { + Predicate predicate = builder.equal(field, 7); + assertThat( + GlobalIndexQuery.create( + rowType, predicate, Collections.singletonList(multi), paths)) + .isNull(); + Range dedicatedRange = new Range(20 + field * 20, 39 + field * 20); + IndexFileMeta dedicated = + indexFile( + "btree", + "single-" + field, + dedicatedRange.from, + dedicatedRange.to, + field, + null); + GlobalIndexQuery query = + GlobalIndexQuery.create( + rowType, predicate, Arrays.asList(multi, dedicated), paths); + assertThat(query).isNotNull(); + assertThat(query.coveredRanges()).containsExactly(dedicatedRange); + assertThat(query.hasCompositeQuery()).isFalse(); + } + } + private IndexFileMeta indexFile( String type, String name, long start, long end, int fieldId, int[] extraFields) { return new IndexFileMeta( diff --git a/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexScannerTest.java b/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexScannerTest.java index 524f54a85c46..8cdffd82e37d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexScannerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexScannerTest.java @@ -128,7 +128,7 @@ private void createIndex(PartitionPredicate partitionPredicate) throws Exception FileStoreTable table = getTableDefault(); SortedGlobalIndexScanner builder = new SortedGlobalIndexScanner(table, "btree"); - builder.withIndexField("f0"); + builder.withIndexFields(Collections.singletonList("f0")); builder.withPartitionPredicate(partitionPredicate); ScanResult scanResult = builder.scan() @@ -212,7 +212,7 @@ public void testScanResultContainsScanSnapshot() throws Exception { ScanResult scanResult = new SortedGlobalIndexScanner(table, "btree") - .withIndexField("f0") + .withIndexFields(Collections.singletonList("f0")) .scan() .orElseThrow(() -> new IllegalStateException("Expected scan result.")); @@ -226,7 +226,7 @@ public void testIncrementalScanNoData() throws Exception { FileStoreTable table = getTableDefault(); SortedGlobalIndexScanner builder = new SortedGlobalIndexScanner(table, "btree"); - builder.withIndexField("f0"); + builder.withIndexFields(Collections.singletonList("f0")); Assertions.assertFalse( builder.incrementalScan().isPresent(), @@ -240,7 +240,7 @@ public void testIncrementalScanAllIndexed() throws Exception { FileStoreTable table = getTableDefault(); SortedGlobalIndexScanner builder = new SortedGlobalIndexScanner(table, "btree"); - builder.withIndexField("f0"); + builder.withIndexFields(Collections.singletonList("f0")); Assertions.assertFalse( builder.incrementalScan().isPresent(), @@ -270,7 +270,7 @@ public void testIncrementalScanWithNewData() throws Exception { table = getTableDefault(); SortedGlobalIndexScanner builder = new SortedGlobalIndexScanner(table, "btree"); - builder.withIndexField("f0"); + builder.withIndexFields(Collections.singletonList("f0")); Optional> incrementalScan = builder.incrementalScan(); Assertions.assertTrue( @@ -311,7 +311,11 @@ public void testIncrementalScanIgnoresNonIndexColumnCompactionAfterSnapshotExpir assertThat(table.snapshotManager().earliestSnapshotId()) .isEqualTo(table.snapshotManager().latestSnapshotId()); - assertThat(dataEvolutionScanner(table).withIndexField("f0").incrementalScan()).isEmpty(); + assertThat( + dataEvolutionScanner(table) + .withIndexFields(Collections.singletonList("f0")) + .incrementalScan()) + .isEmpty(); } @Test @@ -320,13 +324,16 @@ public void testIncrementalScanRefreshesOnlyIndexColumnCompaction() throws Excep createIndex(null); updateColumnAndCompact("f1", 1); - assertThat(dataEvolutionScanner(getTableDefault()).withIndexField("f0").incrementalScan()) + assertThat( + dataEvolutionScanner(getTableDefault()) + .withIndexFields(Collections.singletonList("f0")) + .incrementalScan()) .isEmpty(); DataFileMeta compacted = updateColumnAndCompact("f0", 2); ScanResult scanResult = dataEvolutionScanner(getTableDefault()) - .withIndexField("f0") + .withIndexFields(Collections.singletonList("f0")) .incrementalScan() .orElseThrow( () -> @@ -439,7 +446,7 @@ public void testIncrementalScanWithPartitionPredicate() throws Exception { PartitionPredicate.createPartitionPredicate( partType, Collections.singletonMap("dt", BinaryString.fromString("p0"))); SortedGlobalIndexScanner builder = new SortedGlobalIndexScanner(table, "btree"); - builder.withIndexField("f0"); + builder.withIndexFields(Collections.singletonList("f0")); builder.withPartitionPredicate(PartitionPredicate.fromPredicate(partType, predicate)); List splits = @@ -482,7 +489,8 @@ public void testIncrementalScanRefreshesIndexAfterColumnUpdate() throws Exceptio updateColumn(table, 0L, 5); SortedGlobalIndexScanner scanner = - new SortedGlobalIndexScanner(table, "btree").withIndexField("f0"); + new SortedGlobalIndexScanner(table, "btree") + .withIndexFields(Collections.singletonList("f0")); ScanResult scanResult = scanner.incrementalScan() .orElseThrow( @@ -616,7 +624,8 @@ public void testScanFiltersBlobFilesByManifestEntryFilter() throws Exception { "Test table should contain blob manifest entries."); SortedGlobalIndexScanner builder = - new SortedGlobalIndexScanner(table, "btree").withIndexField("f0"); + new SortedGlobalIndexScanner(table, "btree") + .withIndexFields(Collections.singletonList("f0")); assertNoBlobFiles( builder.scan() .map(ScanResult::entries) diff --git a/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexTestUtils.java b/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexTestUtils.java index d25f68cda6df..58a98ade99b5 100644 --- a/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexTestUtils.java +++ b/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexTestUtils.java @@ -20,6 +20,7 @@ import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; +import org.apache.paimon.globalindex.CompositeKeySerializer; import org.apache.paimon.globalindex.GlobalIndexKeyExtractor; import org.apache.paimon.globalindex.KeySerializer; import org.apache.paimon.reader.RecordReader; @@ -27,7 +28,6 @@ import org.apache.paimon.table.SpecialFields; import org.apache.paimon.table.sink.CommitMessage; import org.apache.paimon.table.source.DataSplit; -import org.apache.paimon.types.DataField; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.CloseableIterator; import org.apache.paimon.utils.InternalRowUtils; @@ -35,7 +35,6 @@ import org.apache.paimon.utils.Range; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -54,13 +53,30 @@ public static List buildIndex( DataSplit dataSplit, long scanSnapshotId) throws Exception { + return buildIndex( + table, + indexType, + Collections.singletonList(indexFieldName), + dataSplit, + scanSnapshotId); + } + + public static List buildIndex( + FileStoreTable table, + String indexType, + List indexFieldNames, + DataSplit dataSplit, + long scanSnapshotId) + throws Exception { SortedGlobalIndexWriter writer = - new SortedGlobalIndexWriter(table, indexType).withIndexField(indexFieldName); - DataField indexField = table.rowType().getField(indexFieldName); - RowType readRowType = - SpecialFields.rowTypeWithRowId(table.rowType()) - .project(Arrays.asList(indexFieldName, SpecialFields.ROW_ID.name())); - InternalRow.FieldGetter fieldGetter = InternalRow.createFieldGetter(indexField.type(), 0); + new SortedGlobalIndexWriter(table, indexType).withIndexFields(indexFieldNames); + List readFields = new ArrayList<>(indexFieldNames); + readFields.add(SpecialFields.ROW_ID.name()); + RowType readRowType = SpecialFields.rowTypeWithRowId(table.rowType()).project(readFields); + InternalRow.FieldGetter[] getters = new InternalRow.FieldGetter[indexFieldNames.size()]; + for (int i = 0; i < getters.length; i++) { + getters[i] = InternalRow.createFieldGetter(readRowType.getTypeAt(i), i); + } GlobalIndexKeyExtractor keyExtractor = writer.keyExtractor(); List> rows = new ArrayList<>(); try (RecordReader reader = @@ -71,8 +87,17 @@ public static List buildIndex( CloseableIterator iterator = reader.toCloseableIterator()) { while (iterator.hasNext()) { InternalRow row = iterator.next(); - Object value = fieldGetter.getFieldOrNull(row); - long rowId = row.getLong(1); + Object value; + if (getters.length == 1) { + value = getters[0].getFieldOrNull(row); + } else { + GenericRow tuple = new GenericRow(getters.length); + for (int i = 0; i < getters.length; i++) { + tuple.setField(i, getters[i].getFieldOrNull(row)); + } + value = tuple; + } + long rowId = row.getLong(getters.length); keyExtractor.extract( value, key -> @@ -84,7 +109,10 @@ public static List buildIndex( } Comparator comparator = - KeySerializer.create(keyExtractor.keyType()).createComparator(); + (keyExtractor.keyType() instanceof RowType + ? new CompositeKeySerializer((RowType) keyExtractor.keyType()) + : KeySerializer.create(keyExtractor.keyType())) + .createComparator(); rows.sort( (left, right) -> { if (left.getKey() == null) { diff --git a/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriterTest.java b/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriterTest.java index 8d7a443ec477..fb1b90fa6a1a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/globalindex/sorted/SortedGlobalIndexWriterTest.java @@ -136,7 +136,7 @@ public GlobalIndexSingleColumnWriter createWriter() { return activeWriter; } }; - writer.withIndexField("f0"); + writer.withIndexFields(Collections.singletonList("f0")); assertThatThrownBy( () -> diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java index 3c0002c2adeb..601ac71561a7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/IndexManifestFileHandlerTest.java @@ -34,6 +34,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.ArrayList; import java.util.Arrays; @@ -338,6 +340,101 @@ public void testSourceMetaDoesNotDisableRangeValidationForOtherFields() throws E .hasMessageContaining("overlapping row range"); } + @ParameterizedTest + @ValueSource(strings = {"btree", "test-vector-ann", "test-fulltext"}) + public void testDifferentOrderedFieldDefinitionsCoexist(String indexType) throws Exception { + TestAppendFileStore fileStore = + TestAppendFileStore.createAppendStore(tempDir, new HashMap<>()); + IndexManifestFile manifestFile = createIndexManifestFile(fileStore); + IndexManifestFileHandler handler = + new IndexManifestFileHandler(manifestFile, BucketMode.BUCKET_UNAWARE); + List definitions = + Arrays.asList( + indexEntry(indexType, "single", 0, 99, 1), + indexEntry(indexType, "pair", 0, 99, 1, 2), + indexEntry(indexType, "triple", 0, 99, 1, 2, 3), + indexEntry(indexType, "reordered", 0, 99, 1, 3, 2), + indexEntry(indexType, "other-pair", 0, 99, 1, 3)); + String manifest = null; + for (IndexManifestEntry definition : definitions) { + manifest = handler.write(manifest, Arrays.asList(definition)); + } + assertThat(manifestFile.read(manifest)).containsExactlyInAnyOrderElementsOf(definitions); + + String retainedManifest = manifest; + IndexManifestEntry replacement = indexEntry(indexType, "replacement", 50, 149, 1, 2); + assertThatThrownBy(() -> handler.write(retainedManifest, Arrays.asList(replacement))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("pair") + .hasMessageContaining("replacement") + .hasMessageContaining("overlapping row range"); + + String replaced = + handler.write( + manifest, Arrays.asList(definitions.get(1).toDeleteEntry(), replacement)); + List expected = new ArrayList<>(definitions); + expected.set(1, replacement); + assertThat(manifestFile.read(replaced)).containsExactlyInAnyOrderElementsOf(expected); + } + + @ParameterizedTest + @ValueSource(strings = {"btree", "test-vector-ann", "test-fulltext"}) + public void testNullAndEmptyExtraFieldsIdentifyTheSameDefinition(String indexType) + throws Exception { + TestAppendFileStore fileStore = + TestAppendFileStore.createAppendStore(tempDir, new HashMap<>()); + IndexManifestFile manifestFile = createIndexManifestFile(fileStore); + IndexManifestFileHandler handler = + new IndexManifestFileHandler(manifestFile, BucketMode.BUCKET_UNAWARE); + IndexManifestEntry previous = indexEntry(indexType, "null-extra", 0, 99, 1); + String manifest = handler.write(null, Arrays.asList(previous)); + IndexManifestEntry tail = + new IndexManifestEntry( + FileKind.ADD, + BinaryRow.EMPTY_ROW, + 0, + new IndexFileMeta( + indexType, + "empty-extra", + 1L, + 100L, + new GlobalIndexMeta(100, 199, 1, new int[0], null), + null)); + String extended = handler.write(manifest, Arrays.asList(tail)); + assertThat(manifestFile.read(extended)).containsExactlyInAnyOrder(previous, tail); + assertThatThrownBy( + () -> + handler.write( + extended, + Arrays.asList( + indexEntry(indexType, "overlap", 100, 199, 1)))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("empty-extra") + .hasMessageContaining("overlapping row range"); + } + + private IndexManifestEntry indexEntry( + String indexType, String fileName, long start, long end, int... fieldIds) { + return new IndexManifestEntry( + FileKind.ADD, + BinaryRow.EMPTY_ROW, + 0, + new IndexFileMeta( + indexType, + fileName, + 1L, + end - start + 1, + new GlobalIndexMeta( + start, + end, + fieldIds[0], + fieldIds.length == 1 + ? null + : Arrays.copyOfRange(fieldIds, 1, fieldIds.length), + null), + null)); + } + private IndexManifestFile createIndexManifestFile(TestAppendFileStore fileStore) { return new IndexManifestFile.Factory( fileStore.fileIO(), diff --git a/paimon-core/src/test/java/org/apache/paimon/metastore/VisibilityWaitCallbackTest.java b/paimon-core/src/test/java/org/apache/paimon/metastore/VisibilityWaitCallbackTest.java index 3b9581f9c9c5..82a325f86034 100644 --- a/paimon-core/src/test/java/org/apache/paimon/metastore/VisibilityWaitCallbackTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/metastore/VisibilityWaitCallbackTest.java @@ -98,7 +98,8 @@ public void testWaitForGlobalIndexBuildOfNewData() throws Exception { writeFuture.get(10, TimeUnit.SECONDS); SortedGlobalIndexScanner builder = - new SortedGlobalIndexScanner(getTableDefault(), "btree").withIndexField("f1"); + new SortedGlobalIndexScanner(getTableDefault(), "btree") + .withIndexFields(Collections.singletonList("f1")); assertThat(builder.incrementalScan()).isNotPresent(); } finally { executor.shutdownNow(); @@ -160,7 +161,8 @@ private void writePartitionRows(FileStoreTable table, String partition, int star private void buildIndex(FileStoreTable table, boolean incremental) throws Exception { SortedGlobalIndexScanner builder = - new SortedGlobalIndexScanner(table, "btree").withIndexField("f1"); + new SortedGlobalIndexScanner(table, "btree") + .withIndexFields(Collections.singletonList("f1")); Optional> scan = incremental ? builder.incrementalScan() : builder.scan(); assertThat(scan).isPresent(); @@ -180,7 +182,7 @@ private void buildIndex(FileStoreTable table, boolean incremental) throws Except private void buildPartitionIndex(FileStoreTable table, String partition) throws Exception { SortedGlobalIndexScanner builder = new SortedGlobalIndexScanner(table, "btree") - .withIndexField("f1") + .withIndexFields(Collections.singletonList("f1")) .withPartitionPredicate(partitionPredicate(table, partition)); Optional> scan = builder.scan(); assertThat(scan).isPresent(); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java index 76faf508ab7a..29daa35f0c46 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/BtreeGlobalIndexTableTest.java @@ -855,7 +855,8 @@ public void testUnionAcrossRangesWithMixedFallbackAnswers() throws Exception { private void createIndexIncremental(String fieldName) throws Exception { FileStoreTable table = (FileStoreTable) catalog.getTable(identifier()); SortedGlobalIndexScanner builder = - new SortedGlobalIndexScanner(table, "btree").withIndexField(fieldName); + new SortedGlobalIndexScanner(table, "btree") + .withIndexFields(Collections.singletonList(fieldName)); ScanResult scanResult = builder.incrementalScan() .orElseThrow( @@ -880,7 +881,8 @@ private void createIndex(String fieldName) throws Exception { private void createIndex(String fieldName, List rowRanges) throws Exception { FileStoreTable table = (FileStoreTable) catalog.getTable(identifier()); SortedGlobalIndexScanner builder = - new SortedGlobalIndexScanner(table, "btree").withIndexField(fieldName); + new SortedGlobalIndexScanner(table, "btree") + .withIndexFields(Collections.singletonList(fieldName)); ScanResult scanResult = builder.scan() .orElseThrow( diff --git a/paimon-core/src/test/java/org/apache/paimon/table/CompositeBTreeTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/CompositeBTreeTableTest.java new file mode 100644 index 000000000000..9976fe8479d4 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/table/CompositeBTreeTableTest.java @@ -0,0 +1,837 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.table; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.globalindex.DataEvolutionGlobalIndexCoverage; +import org.apache.paimon.globalindex.DataEvolutionGlobalIndexScanner; +import org.apache.paimon.globalindex.GlobalIndexEvaluator; +import org.apache.paimon.globalindex.IndexQuerySplit; +import org.apache.paimon.globalindex.ScanResult; +import org.apache.paimon.globalindex.sorted.SortedGlobalIndexScanner; +import org.apache.paimon.globalindex.sorted.SortedGlobalIndexTestUtils; +import org.apache.paimon.index.GlobalIndexMeta; +import org.apache.paimon.index.IndexFileMeta; +import org.apache.paimon.io.CompactIncrement; +import org.apache.paimon.io.DataIncrement; +import org.apache.paimon.manifest.IndexManifestEntry; +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.predicate.PredicateBuilder; +import org.apache.paimon.schema.SchemaChange; +import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.sink.CommitMessage; +import org.apache.paimon.table.sink.CommitMessageImpl; +import org.apache.paimon.table.source.DataSplit; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.Split; +import org.apache.paimon.table.source.TableScan; +import org.apache.paimon.utils.InstantiationUtil; +import org.apache.paimon.utils.Range; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Composite lookups must avoid single-column postings and retain uncovered data. */ +class CompositeBTreeTableTest extends DataEvolutionTestBase { + + @Test + void testCoexistsAndAvoidsSingleColumnPostings() throws Exception { + createTableDefault(); + FileStoreTable table = + table().copy(Collections.singletonMap("sorted-index.records-per-file", "13")); + append(table, 0, 100); + build(table, "f1"); + build(table, "f0"); + build(table, "f1", "f0"); + deleteIndexFiles(table, 1); + PredicateBuilder predicates = new PredicateBuilder(table.rowType()); + Predicate query = query(predicates, "category-a", 7); + for (boolean inReader : Arrays.asList(false, true)) { + FileStoreTable configured = configured(table, "full", inReader); + assertThat(read(configured, query)) + .containsExactlyInAnyOrder("p7", "p27", "p47", "p67", "p87"); + // Both OR branches are composite queries; predicate order differs from index order. + Predicate union = PredicateBuilder.or(query, query(predicates, "category-b", 8)); + assertThat(read(configured, union)).hasSize(10).contains("p7", "p18", "p98"); + assertThat(read(configured, query(predicates, "missing", 7))).isEmpty(); + assertThat(read(configured, PredicateBuilder.and(query, predicates.equal(0, 8)))) + .isEmpty(); + assertThat( + read( + configured, + PredicateBuilder.and( + query, + predicates.isNotNull(1), + predicates.startsWith( + 1, BinaryString.fromString("category-")), + predicates.greaterThan(0, 5)))) + .containsExactlyInAnyOrder("p7", "p27", "p47", "p67", "p87"); + assertThat( + read( + configured, + PredicateBuilder.and( + query, + predicates.startsWith( + 1, BinaryString.fromString("missing"))))) + .isEmpty(); + } + try (DataEvolutionGlobalIndexScanner scanner = + DataEvolutionGlobalIndexScanner.create( + table, + table.store().newIndexFileHandler().scanEntries().stream() + .map(IndexManifestEntry::indexFile) + .collect(Collectors.toList())) + .get()) { + assertThat(scanner.scan(query).get().results().toRangeList()) + .containsExactly( + new Range(7, 7), + new Range(27, 27), + new Range(47, 47), + new Range(67, 67), + new Range(87, 87)); + } + } + + @Test + void testPrefersLongestCompleteKey() throws Exception { + createTableDefault(); + FileStoreTable table = table(); + append(table, 0, 100); + build(table, "f1", "f0"); + build(table, "f1", "f0", "f2"); + // The wider posting list remains in the manifest, but reading it must fail. + deleteIndexFiles(table, 2); + PredicateBuilder predicates = new PredicateBuilder(table.rowType()); + Predicate predicate = + PredicateBuilder.and( + predicates.equal(2, BinaryString.fromString("p7")), + query(predicates, "category-a", 7)); + for (boolean inReader : Arrays.asList(false, true)) { + assertThat(read(configured(table, "full", inReader), predicate)).containsExactly("p7"); + } + } + + @Test + void testUnsupportedScalarPredicatesRetainCompositeFallback() throws Exception { + createTableDefault(); + FileStoreTable table = table(); + append(table, 0, 20); + build(table, "f1", "f0"); + append(table, 20, 40); + build(table, "f2"); + PredicateBuilder predicates = new PredicateBuilder(table.rowType()); + Predicate joint = query(predicates, "category-a", 7); + Predicate unsupported = predicates.notLike(2, BinaryString.fromString("x%")); + Predicate conjunction = PredicateBuilder.and(joint, unsupported); + Predicate union = + PredicateBuilder.or(joint, predicates.notLike(2, BinaryString.fromString("p1%"))); + FileStoreTable withoutIndex = + table.copy( + Collections.singletonMap(CoreOptions.GLOBAL_INDEX_ENABLED.key(), "false")); + for (boolean inReader : Arrays.asList(false, true)) { + assertThat(read(configured(table, "fast", inReader), conjunction)) + .containsExactlyInAnyOrderElementsOf( + inReader + ? Arrays.asList("p7", "p27") + : Collections.singletonList("p7")); + for (String mode : Arrays.asList("full", "detail")) { + FileStoreTable configured = configured(table, mode, inReader); + if (inReader) { + assertThat( + configured + .newReadBuilder() + .withFilter(conjunction) + .newScan() + .plan() + .splits()) + .isNotEmpty() + .allMatch(IndexQuerySplit.class::isInstance); + assertThat( + configured + .newReadBuilder() + .withFilter(union) + .newScan() + .plan() + .splits()) + .isNotEmpty() + .allMatch(IndexQuerySplit.class::isInstance); + } + assertThat(read(configured, conjunction)).containsExactlyInAnyOrder("p7", "p27"); + assertThat(read(configured, union)) + .containsExactlyInAnyOrderElementsOf(read(withoutIndex, union)); + FileStoreTable budgeted = + configured.copy( + Collections.singletonMap( + "btree-index.fallback-scan-max-size", "0 b")); + assertThat( + read( + budgeted, + PredicateBuilder.and( + joint, + predicates.contains( + 2, BinaryString.fromString("7"))))) + .containsExactlyInAnyOrder("p7", "p27"); + } + } + } + + @Test + void testPartialResidualIndexCoverageAcrossSplits() throws Exception { + createTableDefault(); + FileStoreTable table = + table().copy(Collections.singletonMap("source.split.target-size", "1 b")); + append(table, 0, 20); + build(table, "f2"); + append(table, 20, 40); + build(table, "f1", "f0"); + PredicateBuilder predicates = new PredicateBuilder(table.rowType()); + Predicate joint = query(predicates, "category-a", 7); + Predicate unsupported = + PredicateBuilder.and(joint, predicates.notLike(2, BinaryString.fromString("x%"))); + Predicate supported = + PredicateBuilder.and(joint, predicates.startsWith(2, BinaryString.fromString("p"))); + for (boolean inReader : Arrays.asList(false, true)) { + FileStoreTable configured = configured(table, "fast", inReader); + assertThat(read(configured, unsupported)).containsExactlyInAnyOrder("p7", "p27"); + // The composite index covers the tail; the missing scalar group cannot rule it out. + assertThat(read(configured, supported)) + .containsExactlyInAnyOrderElementsOf( + inReader + ? Arrays.asList("p7", "p27") + : Collections.singletonList("p7")); + FileStoreTable budgeted = + configured.copy( + Collections.singletonMap("btree-index.fallback-scan-max-size", "0 b")); + assertThat( + read( + budgeted, + PredicateBuilder.and( + joint, + predicates.contains(2, BinaryString.fromString("p"))))) + .containsExactlyInAnyOrder("p7", "p27"); + for (String mode : Arrays.asList("full", "detail")) { + assertThat(read(configured(table, mode, inReader), supported)) + .containsExactlyInAnyOrder("p7", "p27"); + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"fast", "full", "detail"}) + void testReaderFallbackWithMatchingIndexCoverage(String mode) throws Exception { + createTableDefault(); + FileStoreTable table = configured(table(), mode, true); + append(table, 0, 40); + build(table, "f1", "f0"); + build(table, "f2"); + PredicateBuilder predicates = new PredicateBuilder(table.rowType()); + Predicate joint = query(predicates, "category-a", 7); + FileStoreTable budgeted = + table.copy(Collections.singletonMap("btree-index.fallback-scan-max-size", "0 b")); + FileStoreTable withoutIndex = + table.copy( + Collections.singletonMap(CoreOptions.GLOBAL_INDEX_ENABLED.key(), "false")); + for (Predicate residual : + Arrays.asList( + predicates.notLike(2, BinaryString.fromString("p1%")), + predicates.contains(2, BinaryString.fromString("3")))) { + for (Predicate predicate : + Arrays.asList( + PredicateBuilder.and(joint, residual), + PredicateBuilder.or(joint, residual))) { + assertThat( + budgeted.newReadBuilder() + .withFilter(predicate) + .newScan() + .plan() + .splits()) + .isNotEmpty() + .allMatch(IndexQuerySplit.class::isInstance); + assertThat(read(budgeted, predicate)) + .containsExactlyInAnyOrderElementsOf(read(withoutIndex, predicate)); + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"fast", "full", "detail"}) + void testReaderKeepsCandidatesOutsideResidualCoverage(String mode) throws Exception { + createTableDefault(); + FileStoreTable table = table(); + append(table, 0, 20); + build(table, "f2"); + append(table, 20, 40); + build(table, "f1", "f0"); + PredicateBuilder predicates = new PredicateBuilder(table.rowType()); + Predicate joint = query(predicates, "category-a", 7); + FileStoreTable withoutIndex = + table.copy( + Collections.singletonMap(CoreOptions.GLOBAL_INDEX_ENABLED.key(), "false")); + for (String targetSize : Arrays.asList("1 b", "1 gb")) { + FileStoreTable configured = + configured(table, mode, true) + .copy(Collections.singletonMap("source.split.target-size", targetSize)); + // The scalar metadata proves no match only in the first twenty rows. + for (Predicate residual : + Arrays.asList( + predicates.equal(2, BinaryString.fromString("p27")), + predicates.notLike(2, BinaryString.fromString("p1%")))) { + for (Predicate predicate : + Arrays.asList( + PredicateBuilder.and(joint, residual), + PredicateBuilder.or(joint, residual))) { + assertThat( + configured + .newReadBuilder() + .withFilter(predicate) + .newScan() + .plan() + .splits()) + .isNotEmpty() + .allMatch(IndexQuerySplit.class::isInstance); + assertThat(read(configured, predicate)) + .containsExactlyInAnyOrderElementsOf(read(withoutIndex, predicate)); + } + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"full", "detail"}) + void testPublicScannerFallbackUsesActualSupportedCoverage(String mode) throws Exception { + createTableDefault(); + FileStoreTable table = table(); + append(table, 0, 20); + build(table, "f2"); + append(table, 20, 40); + build(table, "f2"); + build(table, "f1", "f0"); + List files = new ArrayList<>(); + for (IndexManifestEntry entry : table.store().newIndexFileHandler().scanEntries()) { + IndexFileMeta file = entry.indexFile(); + if (file.globalIndexMeta().indexFieldId() == 2 + && file.globalIndexMeta().rowRangeStart() == 20) { + // Model a residual shard over the scan budget without creating a large fixture. + file = + new IndexFileMeta( + file.indexType(), + file.fileName(), + 1_000_000_000L, + file.rowCount(), + file.globalIndexMeta(), + null); + } + files.add(file); + } + FileStoreTable configured = + configured(table, mode, false) + .copy( + Collections.singletonMap( + "btree-index.fallback-scan-max-size", "16 kb")); + PredicateBuilder predicates = new PredicateBuilder(configured.rowType()); + Predicate query = + PredicateBuilder.and( + query(predicates, "category-a", 7), + predicates.contains(2, BinaryString.fromString("p"))); + try (DataEvolutionGlobalIndexScanner scanner = + DataEvolutionGlobalIndexScanner.create(configured, files).get()) { + assertThat(scanner.scan(query).get().results().toRangeList()) + .containsExactly(new Range(7, 7)); + assertThat(scanner.unindexedRows(query).results().toRangeList()) + .containsExactly(new Range(20, 39)); + } + } + + @ParameterizedTest + @ValueSource(strings = {"btree", "test-scalar"}) + void testScalarReadersDoNotOpenMultiFieldIndexFiles(String indexType) throws Exception { + createTableDefault(); + FileStoreTable table = configured(table(), "full", false); + append(table, 0, 40); + List files = + Collections.singletonList( + new IndexFileMeta( + indexType, + "missing-multi-field-index", + 1, + 40, + new GlobalIndexMeta(0, 39, 0, new int[] {1}, null), + null)); + PredicateBuilder predicates = new PredicateBuilder(table.rowType()); + try (DataEvolutionGlobalIndexScanner scanner = + DataEvolutionGlobalIndexScanner.create(table, files).get()) { + for (Predicate predicate : + Arrays.asList( + predicates.equal(0, 7), + predicates.equal(1, BinaryString.fromString("category-a")))) { + assertThat(scanner.scan(predicate)).isEmpty(); + assertThat(scanner.unindexedRows(predicate).results().toRangeList()) + .containsExactly(new Range(0, 39)); + } + } + assertThat(DataEvolutionGlobalIndexScanner.createForScalarFilters(table, null, null, files)) + .isEmpty(); + } + + @Test + void testCompositeQueryDoesNotOpenMultiFieldResidualIndex() throws Exception { + createTableDefault(); + FileStoreTable table = configured(table(), "full", false); + append(table, 0, 40); + build(table, "f1", "f0"); + List files = + table.store().newIndexFileHandler().scanEntries().stream() + .map(IndexManifestEntry::indexFile) + .collect(Collectors.toList()); + files.add( + new IndexFileMeta( + "test-scalar", + "missing-multi-field-residual", + 1, + 40, + new GlobalIndexMeta(0, 39, 2, new int[] {0}, null), + null)); + PredicateBuilder predicates = new PredicateBuilder(table.rowType()); + Predicate predicate = + PredicateBuilder.and( + query(predicates, "category-a", 7), + predicates.equal(2, BinaryString.fromString("p7"))); + try (DataEvolutionGlobalIndexScanner scanner = + DataEvolutionGlobalIndexScanner.create(table, files).get()) { + GlobalIndexEvaluator.Evaluation evaluation = scanner.scanWithCoverage(predicate).get(); + assertThat(evaluation.result().results().toRangeList()) + .containsExactly(new Range(7, 7), new Range(27, 27)); + assertThat(evaluation.contributingFieldIds()).containsExactlyInAnyOrder(0, 1); + assertThat(evaluation.coveredRanges()).containsExactly(new Range(0, 39)); + assertThat(scanner.unindexedRows(predicate).results().isEmpty()).isTrue(); + } + } + + @ParameterizedTest + @CsvSource({"btree,full", "btree,detail", "test-scalar,full", "test-scalar,detail"}) + void testMultiFieldIndexesDoNotSupplySingleFieldCoverage(String indexType, String mode) + throws Exception { + createTableDefault(); + FileStoreTable table = configured(table(), mode, false); + append(table, 0, 40); + List files = + Arrays.asList( + new IndexFileMeta( + indexType, + "multi", + 1, + 40, + new GlobalIndexMeta(0, 39, 0, new int[] {1}, null), + null), + new IndexFileMeta( + indexType, + "primary", + 1, + 10, + new GlobalIndexMeta(0, 9, 0, null, null), + null), + new IndexFileMeta( + indexType, + "extra", + 1, + 10, + new GlobalIndexMeta(20, 29, 1, new int[0], null), + null)); + DataEvolutionGlobalIndexCoverage coverage = + new DataEvolutionGlobalIndexCoverage( + table, + table.snapshotManager().latestSnapshot(), + null, + files, + table.coreOptions().scalarIndexSearchMode()); + assertThat(coverage.unindexedRanges(0)).containsExactly(new Range(10, 39)); + assertThat(coverage.unindexedRanges(1)) + .containsExactly(new Range(0, 19), new Range(30, 39)); + assertThat(coverage.unindexedRanges(Arrays.asList(0, 1), null)) + .containsExactly(new Range(0, 39)); + assertThat( + coverage.unindexedRangesFromCoverage( + Collections.singletonList(new Range(0, 19)), null)) + .containsExactly(new Range(20, 39)); + } + + @ParameterizedTest + @ValueSource(strings = {"full", "detail"}) + void testDroppedDefinitionDoesNotCoverSurvivingExtraColumn(String mode) throws Exception { + createTableDefault(); + FileStoreTable table = table(); + append(table, 0, 40); + IndexFileMeta stale = + new IndexFileMeta( + "test-scalar", + "stale", + 1, + 40, + new GlobalIndexMeta(0, 39, 0, new int[] {1}, null), + null); + IndexFileMeta valid = + new IndexFileMeta( + "test-scalar", + "valid", + 1, + 20, + new GlobalIndexMeta(0, 19, 1, null, null), + null); + catalog.alterTable(identifier(), SchemaChange.dropColumn("f0"), false); + FileStoreTable evolved = configured(table(), mode, false); + DataEvolutionGlobalIndexCoverage coverage = + new DataEvolutionGlobalIndexCoverage( + evolved, + evolved.snapshotManager().latestSnapshot(), + null, + Arrays.asList(stale, valid), + evolved.coreOptions().scalarIndexSearchMode()); + assertThat(coverage.unindexedRanges(1)).containsExactly(new Range(20, 39)); + } + + @ParameterizedTest + @CsvSource({"f0,false", "f0,true", "f1,false", "f1,true"}) + void testDroppedCompositeComponentFallsBack(String dropped, boolean scalar) throws Exception { + createTableDefault(); + FileStoreTable table = table(); + append(table, 0, 40); + build(table, "f1", "f0"); + String surviving = dropped.equals("f0") ? "f1" : "f0"; + if (scalar) { + build(table, surviving); + } + catalog.alterTable(identifier(), SchemaChange.dropColumn(dropped), false); + FileStoreTable evolved = table(); + PredicateBuilder predicates = new PredicateBuilder(evolved.rowType()); + Predicate query = + predicates.equal( + evolved.rowType().getFieldIndex(surviving), + surviving.equals("f0") ? 7 : BinaryString.fromString("category-a")); + FileStoreTable withoutIndex = + evolved.copy( + Collections.singletonMap(CoreOptions.GLOBAL_INDEX_ENABLED.key(), "false")); + List expected = read(withoutIndex, query); + assertThat(expected).hasSize(surviving.equals("f0") ? 4 : 20); + for (boolean inReader : Arrays.asList(false, true)) { + for (String mode : Arrays.asList("fast", "full", "detail")) { + assertThat(read(configured(evolved, mode, inReader), query)) + .containsExactlyInAnyOrderElementsOf(expected); + } + } + Optional scanner = + DataEvolutionGlobalIndexScanner.create( + evolved, + evolved.store().newIndexFileHandler().scanEntries().stream() + .map(IndexManifestEntry::indexFile) + .collect(Collectors.toList())); + assertThat(scanner.isPresent()).isEqualTo(scalar); + if (scanner.isPresent()) { + scanner.get().close(); + } + } + + @Test + void testDroppedUnrelatedScalarDoesNotBreakCompositeLookup() throws Exception { + createTableDefault(); + FileStoreTable table = table(); + append(table, 0, 40); + build(table, "f0"); + build(table, "f1", "f2"); + catalog.alterTable(identifier(), SchemaChange.dropColumn("f0"), false); + FileStoreTable evolved = table(); + PredicateBuilder predicates = new PredicateBuilder(evolved.rowType()); + Predicate query = + PredicateBuilder.and( + predicates.equal(0, BinaryString.fromString("category-a")), + predicates.equal(1, BinaryString.fromString("p7"))); + try (DataEvolutionGlobalIndexScanner scanner = + DataEvolutionGlobalIndexScanner.create( + evolved, + evolved.store().newIndexFileHandler().scanEntries().stream() + .map(IndexManifestEntry::indexFile) + .collect(Collectors.toList())) + .get()) { + assertThat(scanner.scan(query).get().results().toRangeList()) + .containsExactly(new Range(7, 7)); + } + for (boolean inReader : Arrays.asList(false, true)) { + for (String mode : Arrays.asList("fast", "full", "detail")) { + assertThat(read(configured(evolved, mode, inReader), query)).containsExactly("p7"); + } + } + } + + @Test + void testPartialCompositeCoverageDoesNotBorrowSingleColumnCoverage() throws Exception { + createTableDefault(); + FileStoreTable table = table(); + append(table, 0, 20); + build(table, "f1", "f0"); + append(table, 20, 40); + build(table, "f1"); + build(table, "f0"); + deleteIndexFiles(table, 1); + Predicate query = query(new PredicateBuilder(table.rowType()), "category-a", 7); + for (boolean inReader : Arrays.asList(false, true)) { + assertThat(read(configured(table, "fast", inReader), query)).containsExactly("p7"); + for (String mode : Arrays.asList("full", "detail")) { + assertThat(read(configured(table, mode, inReader), query)) + .containsExactlyInAnyOrder("p7", "p27"); + } + } + } + + @Test + void testPartialSingleCoverageDoesNotBorrowCompositeCoverage() throws Exception { + createTableDefault(); + FileStoreTable table = table(); + append(table, 0, 20); + build(table, "f1"); + append(table, 20, 40); + build(table, "f1", "f0"); + Predicate scalar = + new PredicateBuilder(table.rowType()) + .equal(1, BinaryString.fromString("category-a")); + for (boolean inReader : Arrays.asList(false, true)) { + assertThat(read(configured(table, "fast", inReader), scalar)).hasSize(10); + for (String mode : Arrays.asList("full", "detail")) { + assertThat(read(configured(table, mode, inReader), scalar)) + .hasSize(20) + .contains("p27"); + } + } + } + + @Test + void testIncrementalBuildAndRefreshOfEitherComponent() throws Exception { + createTableDefault(); + FileStoreTable table = + table().copy( + Collections.singletonMap( + CoreOptions.GLOBAL_INDEX_COLUMN_UPDATE_ACTION.key(), + "IGNORE")); + append(table, 0, 20); + build(table, "f1", "f0"); + append(table, 20, 40); + build(table, "f1", "f0"); + assertThat( + new SortedGlobalIndexScanner(table, "btree") + .withIndexFields(Arrays.asList("f1", "f0")) + .incrementalScan()) + .isEmpty(); + update(table, "f0", 8); + build(table, "f1", "f0"); + PredicateBuilder predicates = new PredicateBuilder(table.rowType()); + assertThat(read(configured(table, "fast", false), query(predicates, "category-a", 7))) + .containsExactly("p27"); + assertThat(read(configured(table, "fast", true), query(predicates, "category-a", 8))) + .containsExactlyInAnyOrder("p7", "p8", "p28"); + update(table, "f1", BinaryString.fromString("changed")); + build(table, "f1", "f0"); + assertThat(read(configured(table, "fast", true), query(predicates, "changed", 8))) + .containsExactly("p7"); + assertThat(read(configured(table, "fast", false), query(predicates, "category-a", 8))) + .containsExactlyInAnyOrder("p8", "p28"); + } + + @Test + void testThreeFieldsNullComponentsAndUnsupportedPartialPredicate() throws Exception { + createTableDefault(); + FileStoreTable table = + table().copy(Collections.singletonMap("btree-index.bloom-filter.enabled", "true")); + BatchWriteBuilder writes = table.newBatchWriteBuilder(); + try (BatchTableWrite write = writes.newWrite(); + BatchTableCommit commit = writes.newCommit()) { + write.write( + GenericRow.of( + -1, + BinaryString.fromString("category-a"), + BinaryString.fromString("a\u0000b"))); + write.write( + GenericRow.of( + -1, + BinaryString.fromString("category-a"), + BinaryString.fromString(""))); + write.write(GenericRow.of(-1, null, BinaryString.fromString("null-category"))); + write.write( + GenericRow.of( + null, + BinaryString.fromString("category-a"), + BinaryString.fromString("null-number"))); + commit.commit(write.prepareCommit()); + } + build(table, "f1", "f0", "f2"); + PredicateBuilder predicates = new PredicateBuilder(table.rowType()); + Predicate full = + PredicateBuilder.and( + query(predicates, "category-a", -1), + predicates.equal(2, BinaryString.fromString("a\u0000b"))); + for (boolean inReader : Arrays.asList(false, true)) { + assertThat(read(configured(table, "full", inReader), full)).containsExactly("a\u0000b"); + // A partial tuple must use the ordinary data scan, never serialize a scalar as a tuple. + assertThat(read(configured(table, "full", inReader), predicates.equal(0, -1))) + .hasSize(3); + } + } + + private FileStoreTable table() throws Exception { + return (FileStoreTable) catalog.getTable(identifier()); + } + + private Predicate query(PredicateBuilder builder, String category, int itemNumber) { + return PredicateBuilder.and( + builder.equal(0, itemNumber), builder.equal(1, BinaryString.fromString(category))); + } + + private FileStoreTable configured(FileStoreTable table, String mode, boolean inReader) { + Map options = new HashMap<>(); + options.put(CoreOptions.SCALAR_INDEX_SEARCH_MODE.key(), mode); + options.put( + CoreOptions.GLOBAL_INDEX_QUERY_IN_READER_ENABLED.key(), String.valueOf(inReader)); + return table.copy(options); + } + + private List read(FileStoreTable table, Predicate predicate) throws Exception { + ReadBuilder builder = table.newReadBuilder().withFilter(predicate); + TableScan.Plan plan = builder.newScan().plan(); + // Exercise split checkpoint serialization as well as direct execution. + List splits = new ArrayList<>(); + for (Split split : plan.splits()) { + splits.add( + InstantiationUtil.deserializeObject( + InstantiationUtil.serializeObject(split), getClass().getClassLoader())); + } + List values = new ArrayList<>(); + builder.newRead() + .executeFilter() + .createReader(splits) + .forEachRemaining( + row -> + values.add( + row.getString(table.rowType().getFieldIndex("f2")) + .toString())); + return values; + } + + private void append(FileStoreTable table, int from, int to) throws Exception { + BatchWriteBuilder writes = table.newBatchWriteBuilder(); + try (BatchTableWrite write = writes.newWrite(); + BatchTableCommit commit = writes.newCommit()) { + for (int i = from; i < to; i++) { + write.write( + GenericRow.of( + i % 10, + BinaryString.fromString( + (i / 10) % 2 == 0 ? "category-a" : "category-b"), + BinaryString.fromString("p" + i))); + } + commit.commit(write.prepareCommit()); + } + } + + private void update(FileStoreTable table, String field, Object value) throws Exception { + BatchWriteBuilder writes = table.newBatchWriteBuilder(); + try (BatchTableWrite write = + writes.newWrite() + .withWriteType( + table.rowType().project(Collections.singletonList(field))); + BatchTableCommit commit = writes.newCommit()) { + for (int i = 0; i < 20; i++) { + Object original = + field.equals("f0") + ? i % 10 + : BinaryString.fromString( + (i / 10) % 2 == 0 ? "category-a" : "category-b"); + write.write(GenericRow.of(i == 7 ? value : original)); + } + List messages = write.prepareCommit(); + setFirstRowId(messages, 0L); + commit.commit(messages); + } + } + + private void build(FileStoreTable table, String... fields) throws Exception { + List names = Arrays.asList(fields); + Optional> scan = + new SortedGlobalIndexScanner(table, "btree") + .withIndexFields(names) + .incrementalScan(); + if (!scan.isPresent()) { + return; + } + ScanResult result = scan.get(); + List messages = new ArrayList<>(); + for (DataSplit split : result.entries()) { + messages.addAll( + SortedGlobalIndexTestUtils.buildIndex( + table, "btree", names, split, result.scanSnapshotId())); + } + for (IndexManifestEntry entry : result.deletedIndexEntries()) { + messages.add( + new CommitMessageImpl( + entry.partition(), + entry.bucket(), + null, + DataIncrement.deleteIndexIncrement( + Collections.singletonList(entry.indexFile())), + CompactIncrement.emptyIncrement())); + } + try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) { + commit.commit(messages); + } + } + + private void deleteIndexFiles(FileStoreTable table, int fieldCount) throws Exception { + int deleted = 0; + for (IndexManifestEntry entry : table.store().newIndexFileHandler().scanEntries()) { + IndexFileMeta file = entry.indexFile(); + int[] extraFields = file.globalIndexMeta().extraFieldIds(); + if (1 + (extraFields == null ? 0 : extraFields.length) == fieldCount) { + assertThat( + table.fileIO() + .delete( + table.store() + .pathFactory() + .globalIndexFileFactory() + .toPath(file), + false)) + .isTrue(); + deleted++; + } + } + assertThat(deleted).isPositive(); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/table/IndexQuerySplitTest.java b/paimon-core/src/test/java/org/apache/paimon/table/IndexQuerySplitTest.java index 0dff41970940..a7ca6b9eb267 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/IndexQuerySplitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/IndexQuerySplitTest.java @@ -294,10 +294,16 @@ public void testResultsAcrossModesAndPredicates(String indexType) throws Excepti .newScan() .plan(); List actual = read(read, indexQueryPlan.splits()); + List expected = read(read, read.newScan().plan().splits()); + // In the Reader, f2 covers the tail where f1 has no local index group. + if (mode.equals("fast") && predicate.equals(predicates.get(1))) { + expected = Arrays.asList(50, 150, 250); + } else if (mode.equals("fast") && predicate.equals(predicates.get(2))) { + expected = Arrays.asList(50, 150, 250, 251); + } assertThat(actual) .as("%s / %s / %s", indexType, mode, predicate) - .containsExactlyInAnyOrderElementsOf( - read(read, read.newScan().plan().splits())); + .containsExactlyInAnyOrderElementsOf(expected); if (!mode.equals("fast")) { ReadBuilder full = configured @@ -527,7 +533,7 @@ public void testPrecomputedRangesAndUnsupportedPredicateKeepExistingPath() throw @ParameterizedTest @ValueSource(strings = {"btree", "bitmap"}) - public void testBudgetFailureIsDeferredToReader(String indexType) throws Exception { + public void testBudgetFallbackIsDeferredToReader(String indexType) throws Exception { write(10); appendRows(10, 1000); FileStoreTable table = smallSplits(getTableDefault()); @@ -550,14 +556,88 @@ public void testBudgetFailureIsDeferredToReader(String indexType) throws Excepti long max = Collections.max(sizes.values()); assertThat(max).isGreaterThan(min); String budgetKey = indexType + "-index.fallback-scan-max-size"; - table = table.copy(Collections.singletonMap(budgetKey, min + " b")); + Map options = new HashMap<>(); + options.put(budgetKey, min + " b"); + options.put(CoreOptions.GLOBAL_INDEX_SEARCH_MODE.key(), "full"); + table = table.copy(options); Predicate predicate = new PredicateBuilder(table.rowType()).contains(1, str("5")); ReadBuilder read = table.newReadBuilder().withFilter(predicate); ReadBuilder indexQueryRead = distributedTable(table).newReadBuilder().withFilter(predicate); List splits = indexQueryRead.newScan().plan().splits(); assertThat(splits).isNotEmpty().allMatch(IndexQuerySplit.class::isInstance); - assertThat(read(read, read.newScan().plan().splits())).isNotEmpty(); - assertThatThrownBy(() -> read(indexQueryRead, splits)).isInstanceOf(IOException.class); + ReadBuilder withoutIndex = + table.copy( + Collections.singletonMap( + CoreOptions.GLOBAL_INDEX_ENABLED.key(), "false")) + .newReadBuilder() + .withFilter(predicate); + List expected = read(withoutIndex, withoutIndex.newScan().plan().splits()); + assertThat(expected).hasSize(271); + assertThat(read(read, read.newScan().plan().splits())) + .containsExactlyInAnyOrderElementsOf(expected); + assertThat(read(indexQueryRead, splits)).containsExactlyInAnyOrderElementsOf(expected); + // One split containing both a supported and an over-budget shard must use actual + // evaluated coverage, rather than the complete metadata coverage from planning. + ReadBuilder combinedRead = + distributedTable( + table.copy( + Collections.singletonMap( + "source.split.target-size", "1 gb"))) + .newReadBuilder() + .withFilter(predicate); + List combinedSplits = combinedRead.newScan().plan().splits(); + assertThat(combinedSplits).hasSize(1).allMatch(IndexQuerySplit.class::isInstance); + assertThat(read(combinedRead, combinedSplits)) + .containsExactlyInAnyOrderElementsOf(expected); + } + + @Test + public void testUnsupportedAlternativeIndexUsesSupportedCandidates() throws Exception { + write(100); + createIndex("btree", "f1"); + createFMIndex(getTableDefault()); + appendRows(100, 200); + FileStoreTable table = + smallSplits(getTableDefault()) + .copy( + Collections.singletonMap( + CoreOptions.GLOBAL_INDEX_SEARCH_MODE.key(), "fast")); + Predicate predicate = new PredicateBuilder(table.rowType()).equal(1, str("a50")); + assertThat(table.newScan().plan().splits()).hasSize(2); + ReadBuilder read = distributedTable(table).newReadBuilder().withFilter(predicate); + List splits = read.newScan().plan().splits(); + assertThat(splits).hasSize(1).allMatch(IndexQuerySplit.class::isInstance); + assertThat(((IndexQuerySplit) splits.get(0)).evaluate(table.fileIO()).rowRanges()) + .containsExactly(new Range(50, 50)); + assertThat(read(read, splits)).containsExactly(50); + } + + @ParameterizedTest + @ValueSource(strings = {"btree", "bitmap"}) + public void testFastUnsupportedPredicateScansCoveredRows(String indexType) throws Exception { + write(100); + createIndex(indexType, "f1"); + appendRows(100, 200); + FileStoreTable table = + smallSplits(getTableDefault()) + .copy( + Collections.singletonMap( + CoreOptions.GLOBAL_INDEX_SEARCH_MODE.key(), "fast")); + Predicate predicate = new PredicateBuilder(table.rowType()).notLike(1, str("a0")); + assertThat(table.newScan().plan().splits()).hasSize(2); + ReadBuilder read = distributedTable(table).newReadBuilder().withFilter(predicate); + List splits = read.newScan().plan().splits(); + assertThat(splits).hasSize(1).allMatch(IndexQuerySplit.class::isInstance); + assertThat(read(read, splits)).hasSize(99).contains(1, 99).doesNotContain(0, 100, 199); + table = + table.copy( + Collections.singletonMap( + indexType + "-index.fallback-scan-max-size", "0 b")); + predicate = new PredicateBuilder(table.rowType()).contains(1, str("5")); + read = distributedTable(table).newReadBuilder().withFilter(predicate); + splits = read.newScan().plan().splits(); + assertThat(splits).hasSize(1).allMatch(IndexQuerySplit.class::isInstance); + assertThat(read(read, splits)).hasSize(19).contains(5, 50, 95).doesNotContain(150, 195); } @ParameterizedTest @@ -827,6 +907,38 @@ private FileStoreTable smallSplits(FileStoreTable table) { public void testFMFallbackAndMixedDistributedIndex() throws Exception { write(100); FileStoreTable table = getTableDefault(); + createFMIndex(table); + Map options = new HashMap<>(); + options.put(CoreOptions.SCALAR_INDEX_SEARCH_MODE.key(), "full"); + options.put(FMGlobalIndexOptions.LOCATE_COST_RATIO.key(), "1"); + table = distributedTable(table.copy(options)); + PredicateBuilder b = new PredicateBuilder(table.rowType()); + Predicate predicate = + PredicateBuilder.and( + b.contains(1, str("5")), + b.contains(1, str("1")), + b.startsWith(2, str("b"))); + ReadBuilder read = table.newReadBuilder().withFilter(predicate); + List fmSplits = read.newScan().plan().splits(); + assertThat(fmSplits).isNotEmpty().allMatch(IndexQuerySplit.class::isInstance); + assertThat(read(read, fmSplits)).containsExactly(15, 51); + + createIndex("btree", "f2"); + List splits = read.newScan().plan().splits(); + assertThat(splits).isNotEmpty().allMatch(IndexQuerySplit.class::isInstance); + assertThat(read(read, splits)).containsExactly(15, 51); + + ReadBuilder orRead = + table.newReadBuilder() + .withFilter( + PredicateBuilder.or( + b.contains(1, str("5")), b.equal(2, str("b10")))); + List orSplits = orRead.newScan().plan().splits(); + assertThat(orSplits).isNotEmpty().allMatch(IndexQuerySplit.class::isInstance); + assertThat(read(orRead, orSplits)).contains(5, 10, 15, 55, 95).hasSize(20); + } + + private void createFMIndex(FileStoreTable table) throws Exception { GlobalIndexFileReadWrite io = new GlobalIndexFileReadWrite( table.fileIO(), table.store().pathFactory().globalIndexFileFactory()); @@ -864,34 +976,6 @@ public void testFMFallbackAndMixedDistributedIndex() throws Exception { DataIncrement.indexIncrement(indexes), CompactIncrement.emptyIncrement()))); } - Map options = new HashMap<>(); - options.put(CoreOptions.SCALAR_INDEX_SEARCH_MODE.key(), "full"); - options.put(FMGlobalIndexOptions.LOCATE_COST_RATIO.key(), "1"); - table = distributedTable(table.copy(options)); - PredicateBuilder b = new PredicateBuilder(table.rowType()); - Predicate predicate = - PredicateBuilder.and( - b.contains(1, str("5")), - b.contains(1, str("1")), - b.startsWith(2, str("b"))); - ReadBuilder read = table.newReadBuilder().withFilter(predicate); - List fmSplits = read.newScan().plan().splits(); - assertThat(fmSplits).isNotEmpty().allMatch(IndexQuerySplit.class::isInstance); - assertThat(read(read, fmSplits)).containsExactly(15, 51); - - createIndex("btree", "f2"); - List splits = read.newScan().plan().splits(); - assertThat(splits).isNotEmpty().allMatch(IndexQuerySplit.class::isInstance); - assertThat(read(read, splits)).containsExactly(15, 51); - - ReadBuilder orRead = - table.newReadBuilder() - .withFilter( - PredicateBuilder.or( - b.contains(1, str("5")), b.equal(2, str("b10")))); - List orSplits = orRead.newScan().plan().splits(); - assertThat(orSplits).isNotEmpty().allMatch(IndexQuerySplit.class::isInstance); - assertThat(read(orRead, orSplits)).contains(5, 10, 15, 55, 95).hasSize(20); } private List read(ReadBuilder read, List splits) throws Exception { @@ -921,7 +1005,8 @@ private void appendRows(int start, int end) throws Exception { private void createIndex(String type, String field) throws Exception { FileStoreTable table = getTableDefault(); SortedGlobalIndexScanner builder = - new SortedGlobalIndexScanner(table, type).withIndexField(field); + new SortedGlobalIndexScanner(table, type) + .withIndexFields(Collections.singletonList(field)); createIndex(type, field, builder.scan().get().entries()); } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/MultiValueGlobalIndexTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/MultiValueGlobalIndexTableTest.java index 90d440a98c2f..f9a894f02cd3 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/MultiValueGlobalIndexTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/MultiValueGlobalIndexTableTest.java @@ -108,7 +108,8 @@ public void testCoreScanUsesMultiValueIndexAndPreservesCoverage() throws Excepti private void buildIndex(FileStoreTable table) throws Exception { SortedGlobalIndexScanner scanner = - new SortedGlobalIndexScanner(table, "multivalue").withIndexField("tags"); + new SortedGlobalIndexScanner(table, "multivalue") + .withIndexFields(Collections.singletonList("tags")); ScanResult scanResult = scanner.incrementalScan() .orElseThrow( diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java index 574d4d588834..630c82b15d0b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/FullTextSearchBuilderTest.java @@ -134,6 +134,64 @@ public void testFullTextSearchEndToEnd() throws Exception { assertThat(ids).containsAnyOf(0, 1, 3); } + @ParameterizedTest + @CsvSource({"full,true", "detail,true", "full,false", "detail,false"}) + public void testPartialCompositeKeepsScalarFilterCoverage(String mode, boolean scalar) + throws Exception { + createTableDefault(); + FileStoreTable table = + getTableDefault() + .copy( + Collections.singletonMap( + CoreOptions.SCALAR_INDEX_SEARCH_MODE.key(), mode)); + String[] documents = {"alpha keyword", "beta keyword", "gamma keyword"}; + writeDocuments(table, documents); + buildAndCommitIndex(table, documents); + if (scalar) { + buildAndCommitIdBTreeIndex(table, documents.length); + buildAndCommitBTreeIndex(table, documents); + } + List fields = + Arrays.asList( + table.rowType().getField(TEXT_FIELD_NAME), table.rowType().getField("id")); + GlobalIndexSingleColumnWriter writer = + (GlobalIndexSingleColumnWriter) + GlobalIndexBuilderUtils.createIndexWriter( + table, "btree", fields, table.coreOptions().toConfiguration()); + writer.write(GenericRow.of(BinaryString.fromString(documents[0]), 0), 0); + List files = + GlobalIndexBuilderUtils.toIndexFileMetas( + table.fileIO(), + table.store().pathFactory().globalIndexFileFactory(), + table.coreOptions(), + new Range(0, 0), + fields, + "btree", + writer.finish(), + null); + try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) { + commit.commit( + Collections.singletonList( + new CommitMessageImpl( + BinaryRow.EMPTY_ROW, + 0, + null, + DataIncrement.indexIncrement(files), + CompactIncrement.emptyIncrement()))); + } + PredicateBuilder predicates = new PredicateBuilder(table.rowType()); + GlobalIndexResult result = + table.newFullTextSearchBuilder() + .withQuery(TEXT_FIELD_NAME, matchQuery("keyword")) + .withLimit(1) + .withFilter( + PredicateBuilder.and( + predicates.equal(0, 1), + predicates.equal(1, BinaryString.fromString(documents[1])))) + .executeLocal(); + assertThat(result.results()).containsExactly(1L); + } + @Test public void testFullTextSearchExcludesDeletedIndexedRows() throws Exception { Identifier identifier = identifier("full_text_deleted_indexed_rows"); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java index 13a1fe39a42c..f56d1fc230fd 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchBuilderTest.java @@ -68,6 +68,8 @@ import org.apache.paimon.utils.RoaringNavigableMap64; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -1434,12 +1436,13 @@ public void testVectorSearchRequiresVectorColumnAsPrimaryField() throws Exceptio assertThat(searchBuilder.executeLocal().results().isEmpty()).isTrue(); } - @Test - public void testVectorPrimaryMultiFieldIndexServesScalarExtraField() throws Exception { + @ParameterizedTest + @ValueSource(strings = {"fast", "full", "detail"}) + public void testMultiFieldIndexDoesNotProvideSingleFieldCoverage(String mode) throws Exception { catalog.createTable( identifier("vector_primary_scalar_extra_field_table"), vectorSchemaBuilder(VECTOR_FIELD_NAME) - .option(CoreOptions.GLOBAL_INDEX_SEARCH_MODE.key(), "full") + .option(CoreOptions.GLOBAL_INDEX_SEARCH_MODE.key(), mode) .build(), false); FileStoreTable table = getTable(identifier("vector_primary_scalar_extra_field_table")); @@ -1462,11 +1465,16 @@ public void testVectorPrimaryMultiFieldIndexServesScalarExtraField() throws Exce .newVectorScan() .scan(); - // A vector-primary multi-field index can serve both the vector search and a scalar - // predicate on an extra field. It must therefore be attached as both vector and scalar - // index input, and full search mode must not create a raw fallback for the covered range. + // The reader can serve both roles, but a multi-field definition does not supply + // single-field coverage. Full/detail mode must retain a data fallback for these rows. assertThat(indexVectorSearchSplits(plan.splits())).hasSize(1); - assertThat(rawVectorSearchSplits(plan.splits())).isEmpty(); + if ("fast".equals(mode)) { + assertThat(rawVectorSearchSplits(plan.splits())).isEmpty(); + } else { + assertThat(rawVectorSearchSplits(plan.splits())).hasSize(1); + assertThat(rawVectorSearchSplits(plan.splits()).get(0).rowRanges()) + .containsExactly(new Range(0, 1)); + } IndexVectorSearchSplit split = indexVectorSearchSplits(plan.splits()).get(0); assertThat(split.vectorIndexFiles()).hasSize(2); @@ -1506,10 +1514,8 @@ public void testScalarExtraFieldComplementsPartialDedicatedIndex() throws Except DataField idField = table.rowType().getField("id"); buildAndCommitVectorIndexWithFields( table, vectors, new Range(0, 3), Arrays.asList(vectorField, idField)); - // The dedicated scalar index covers only the head. The vector index's scalar extra field - // must still serve the tail; otherwise DataEvolutionGlobalIndexScanner's primary-field - // preference drops - // rows [2, 3] even though coverage planning treats them as indexed. + // The dedicated scalar index covers only the head. The multi-field index's scalar extra + // field does not extend that coverage, so the tail remains eligible for a data fallback. buildAndCommitBTreeIndex(table, new int[] {0, 1}, new Range(0, 1)); Predicate idFilter = new PredicateBuilder(table.rowType()).greaterOrEqual(0, 2); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchRowFilterExactnessTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchRowFilterExactnessTest.java index acd97870d832..c0bd13300f8b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchRowFilterExactnessTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/VectorSearchRowFilterExactnessTest.java @@ -50,6 +50,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; import java.util.Arrays; @@ -129,6 +130,49 @@ public void testContainsOnBTreeColumnRanksOnlyMatchingRows() throws Exception { assertThat(result.results()).containsExactly(1L); } + @ParameterizedTest + @CsvSource({"full,true", "detail,true", "full,false", "detail,false"}) + public void testPartialCompositeKeepsScalarFilterCoverage(String mode, boolean indexed) + throws Exception { + FileStoreTable table = + createTable( + "vector_composite_" + mode + indexed, + schemaBuilder(true) + .option(CoreOptions.SCALAR_INDEX_SEARCH_MODE.key(), mode) + .option(CoreOptions.VECTOR_INDEX_SEARCH_MODE.key(), "full")); + String[] names = {"alpha", "beta", "gamma"}; + float[][] vectors = {{1.0f, 0.0f}, {0.6f, 0.8f}, {0.0f, 1.0f}}; + write(table, names, vectors); + buildAndCommitVectorIndex(table, vectors, new Range(0, indexed ? 2 : 0)); + buildAndCommitNameBTreeIndex(table, names); + buildAndCommitIdBTreeIndex(table, names.length); + List fields = + Arrays.asList(table.rowType().getField("name"), table.rowType().getField("id")); + GlobalIndexSingleColumnWriter writer = + (GlobalIndexSingleColumnWriter) + GlobalIndexBuilderUtils.createIndexWriter( + table, "btree", fields, table.coreOptions().toConfiguration()); + writer.write(GenericRow.of(BinaryString.fromString(names[0]), 0), 0); + writer.write(GenericRow.of(BinaryString.fromString(names[1]), 1), 1); + commitIndex( + table, + GlobalIndexBuilderUtils.toIndexFileMetas( + table.fileIO(), + table.store().pathFactory().globalIndexFileFactory(), + table.coreOptions(), + new Range(0, 1), + fields, + "btree", + writer.finish(), + null)); + PredicateBuilder predicates = new PredicateBuilder(table.rowType()); + Predicate query = + PredicateBuilder.and( + predicates.equal(0, 2), + predicates.equal(1, BinaryString.fromString("gamma"))); + assertThat(search(table, query, 1)).containsExactly(2L); + } + @Test public void testPartiallyIndexedConjunctionRanksOnlyMatchingRows() throws Exception { createTableDefault(); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/SortedIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/SortedIndexTopoBuilder.java index c4d86def5297..54c010c22879 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/SortedIndexTopoBuilder.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/SortedIndexTopoBuilder.java @@ -153,7 +153,9 @@ public static Optional> buildIndexStream( List> allStreams = new ArrayList<>(); for (String indexColumn : indexColumns) { SortedGlobalIndexScanner indexScanner = - indexScannerSupplier.get().withIndexField(indexColumn); + indexScannerSupplier + .get() + .withIndexFields(Collections.singletonList(indexColumn)); if (partitionPredicate != null) { indexScanner = indexScanner.withPartitionPredicate(partitionPredicate); } @@ -247,7 +249,7 @@ public static Optional> buildIndexStream( splitTasks, readBuilder, new SortedGlobalIndexWriter(table, indexType, userOptions) - .withIndexField(indexColumn), + .withIndexFields(Collections.singletonList(indexColumn)), scanResult.scanSnapshotId(), partitionFieldSize, taskIdPos, diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/globalindex/SortedIndexTopoBuilderTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/globalindex/SortedIndexTopoBuilderTest.java index cdd5cf31c8fc..f5abf37b70e5 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/globalindex/SortedIndexTopoBuilderTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/globalindex/SortedIndexTopoBuilderTest.java @@ -117,7 +117,8 @@ public void testWriteIndexOperatorClosesActiveWriter() throws Exception { @Test public void testBuildIndexReturnsFalseWhenNoBuildTask() throws Exception { SortedGlobalIndexScanner indexScanner = mock(SortedGlobalIndexScanner.class); - when(indexScanner.withIndexField("id")).thenReturn(indexScanner); + when(indexScanner.withIndexFields(Collections.singletonList("id"))) + .thenReturn(indexScanner); when(indexScanner.incrementalScan()).thenReturn(Optional.empty()); StreamExecutionEnvironment env = mock(StreamExecutionEnvironment.class); @@ -138,7 +139,8 @@ public void testBuildIndexReturnsFalseWhenNoBuildTask() throws Exception { @Test public void testBuildIndexStreamReturnsEmptyWhenNoBuildTask() throws Exception { SortedGlobalIndexScanner indexScanner = mock(SortedGlobalIndexScanner.class); - when(indexScanner.withIndexField("id")).thenReturn(indexScanner); + when(indexScanner.withIndexFields(Collections.singletonList("id"))) + .thenReturn(indexScanner); when(indexScanner.incrementalScan()).thenReturn(Optional.empty()); StreamExecutionEnvironment env = mock(StreamExecutionEnvironment.class); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/VectorSearchProcedureITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/VectorSearchProcedureITCase.java index 1cf45a2fa5b9..544836ee88df 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/VectorSearchProcedureITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/procedure/VectorSearchProcedureITCase.java @@ -686,8 +686,14 @@ public void testDistributedMixedIndexedAndRawVectorSearchWithFilter() throws Exc .map(RawVectorSearchSplit.class::cast) .collect(Collectors.toList()); assertThat(rawSplits).hasSize(1); - assertThat(rawSplits.get(0).rowRanges()).containsExactly(new Range(8, 11)); - assertThat(rawSplits.get(0).scalarIndexFiles()).isEmpty(); + // Multi-field definitions do not supply single-field coverage, so FULL search retains + // these rows on the raw fallback path as well. + assertThat(rawSplits.get(0).rowRanges()).containsExactly(new Range(0, 11)); + assertThat(rawSplits.get(0).scalarIndexFiles()) + .containsExactlyInAnyOrderElementsOf( + indexSplits.stream() + .flatMap(split -> split.vectorIndexFiles().stream()) + .collect(Collectors.toList())); List rows = sql( diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/IndexQuerySourceTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/IndexQuerySourceTest.java index d2b5a76e855d..94290a31b8a2 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/IndexQuerySourceTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/IndexQuerySourceTest.java @@ -181,7 +181,8 @@ private FileStoreTable indexedTable() throws Exception { List commits = new ArrayList<>(); for (String field : new String[] {"f1", "f2"}) { SortedGlobalIndexScanner builder = - new SortedGlobalIndexScanner(table, "btree").withIndexField(field); + new SortedGlobalIndexScanner(table, "btree") + .withIndexFields(Collections.singletonList(field)); for (DataSplit split : builder.scan().get().entries()) { commits.addAll( SortedGlobalIndexTestUtils.buildIndex( diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/sorted/SortedIndexTopoBuilder.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/sorted/SortedIndexTopoBuilder.java index 3b9a125eb128..e1f23c9fa14f 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/sorted/SortedIndexTopoBuilder.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/globalindex/sorted/SortedIndexTopoBuilder.java @@ -103,7 +103,7 @@ public List buildIndex( throws IOException { SortedGlobalIndexScanner indexScanner = new SortedGlobalIndexScanner(table, indexType, options) - .withIndexField(indexField.name()); + .withIndexFields(Collections.singletonList(indexField.name())); if (partitionPredicate != null) { indexScanner = indexScanner.withPartitionPredicate(partitionPredicate); } @@ -145,7 +145,7 @@ public List buildIndex( BinaryRowSerializer binaryRowSerializer = new BinaryRowSerializer(partitionKeyNum); SortedGlobalIndexWriter indexWriter = new SortedGlobalIndexWriter(table, indexType, options) - .withIndexField(indexField.name()); + .withIndexFields(Collections.singletonList(indexField.name())); final byte[] serializedWriter = InstantiationUtil.serializeObject(indexWriter); if (keyExtractor.isIdentity()) { List buildTasks = new ArrayList<>();