From 345b9d18f0e36f0ceec4f0b2b6c93658657ba71d Mon Sep 17 00:00:00 2001 From: ofiryanai Date: Sun, 19 Apr 2026 18:48:23 +0300 Subject: [PATCH 01/64] MOD-14916 Devirtualize HNSW / brute-force search hot path (#937) * MOD-14916 Devirtualize distance + getElement on HNSW search hot path MOD-14916 / LTK perf investigation. Two virtual dispatches per HNSW candidate added between v2.10.21 and v8.2.6 account for a measurable share of the KNN regression observed in the LTK benchmarks (-38% throughput on Intel). Both are removed here with the minimum possible change. V1 - distance computation: Every calcDistance() call goes through IndexCalculatorInterface's vtable to reach DistanceCalculatorCommon, which then calls the underlying SIMD function pointer. The intermediate vtable hop is pure indirection; the concrete calculator class is fixed for the life of an index. Expose the underlying dist_func via a new pure-virtual getDistFunc() on IndexCalculatorInterface, implemented by DistanceCalculatorCommon. Cache the returned function pointer in VecSimIndexAbstract at construction time and call it directly in calcDistance(), bypassing the virtual dispatch. V2 - vector fetch: HNSWIndex::getDataByInternalId and BruteForceIndex::getDataByInternalId call this->vectors->getElement(id), which is virtual through the RawDataContainer base. DataBlocksContainer is the only concrete implementation, and this->vectors is always a DataBlocksContainer (created and owned by VecSimIndexAbstract's constructor). Use a static_cast to DataBlocksContainer* plus a qualified call to DataBlocksContainer::getElement to skip the vtable lookup. No behavior change; per-candidate distance and neighbor-fetch calls on HNSW / brute force search paths become direct function-pointer / direct-member calls. Headers in index_factories, hnsw_serializer, and brute_force_factory compile cleanly. * MOD-14916 Inline DataBlocksContainer::getElement on HNSW search hot path Follow-up to the previous V1/V2 devirt commit. The static_cast+qualified call in getDataByInternalId removed the vtable lookup but left the larger cost on the table: DataBlocksContainer::getElement was still defined in data_blocks_container.cpp, so every per-candidate neighbor fetch still paid a real out-of-line function call and a bounds-checked blocks.at() lookup. Without LTO the compiler could neither inline the body nor hoist the div/mod in the HNSW hot loop. Move the definition into the header as inline and drop the .at() bounds check to match the v2.10.21 baseline, which used unchecked operator[] and was fully inlined into processCandidate. Also add a getDistFunc() override to DistanceCalculatorDummy in test_components.cpp so BUILD_TESTS still compiles after the pure virtual added in the previous commit. (cherry picked from commit 4ca500a6fbac0fde659717f726e8c23442c37e7c) --- src/VecSim/algorithms/brute_force/brute_force.h | 5 ++++- src/VecSim/algorithms/hnsw/hnsw.h | 4 +++- src/VecSim/containers/data_blocks_container.cpp | 5 ----- src/VecSim/containers/data_blocks_container.h | 6 +++++- src/VecSim/spaces/computer/calculator.h | 5 +++++ src/VecSim/vec_sim_index.h | 16 +++++++++++++--- tests/unit/test_components.cpp | 3 +++ 7 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/VecSim/algorithms/brute_force/brute_force.h b/src/VecSim/algorithms/brute_force/brute_force.h index 3be453024..be19f2f4c 100644 --- a/src/VecSim/algorithms/brute_force/brute_force.h +++ b/src/VecSim/algorithms/brute_force/brute_force.h @@ -43,7 +43,10 @@ class BruteForceIndex : public VecSimIndexAbstract { size_t indexCapacity() const override; std::unique_ptr getVectorsIterator() const; const DataType *getDataByInternalId(idType id) const { - return reinterpret_cast(this->vectors->getElement(id)); + // `vectors` is always a DataBlocksContainer; skip the RawDataContainer vtable. + return reinterpret_cast( + static_cast(this->vectors) + ->DataBlocksContainer::getElement(id)); } VecSimQueryReply *topKQuery(const void *queryBlob, size_t k, VecSimQueryParams *queryParams) const override; diff --git a/src/VecSim/algorithms/hnsw/hnsw.h b/src/VecSim/algorithms/hnsw/hnsw.h index e5994d314..1a3776fa2 100644 --- a/src/VecSim/algorithms/hnsw/hnsw.h +++ b/src/VecSim/algorithms/hnsw/hnsw.h @@ -385,7 +385,9 @@ labelType HNSWIndex::getEntryPointLabel() const { template const char *HNSWIndex::getDataByInternalId(idType internal_id) const { - return this->vectors->getElement(internal_id); + // `vectors` is always a DataBlocksContainer; skip the RawDataContainer vtable on the hot path. + return static_cast(this->vectors) + ->DataBlocksContainer::getElement(internal_id); } template diff --git a/src/VecSim/containers/data_blocks_container.cpp b/src/VecSim/containers/data_blocks_container.cpp index bf63b683a..f6010b7db 100644 --- a/src/VecSim/containers/data_blocks_container.cpp +++ b/src/VecSim/containers/data_blocks_container.cpp @@ -38,11 +38,6 @@ RawDataContainer::Status DataBlocksContainer::addElement(const void *element, si return Status::OK; } -const char *DataBlocksContainer::getElement(size_t id) const { - assert(id < element_count); - return blocks.at(id / this->block_size).getElement(id % this->block_size); -} - RawDataContainer::Status DataBlocksContainer::removeElement(size_t id) { assert(id == element_count - 1); // only the last element can be removed blocks.back().popLastElement(); diff --git a/src/VecSim/containers/data_blocks_container.h b/src/VecSim/containers/data_blocks_container.h index c375590f2..74f66de11 100644 --- a/src/VecSim/containers/data_blocks_container.h +++ b/src/VecSim/containers/data_blocks_container.h @@ -40,7 +40,11 @@ class DataBlocksContainer : public VecsimBaseObject, public RawDataContainer { Status addElement(const void *element, size_t id) override; - const char *getElement(size_t id) const override; + // Inlined so the hot search path (via getDataByInternalId) can fold the indexing arithmetic. + const char *getElement(size_t id) const override { + assert(id < element_count); + return blocks[id / block_size].getElement(id % block_size); + } Status removeElement(size_t id) override; diff --git a/src/VecSim/spaces/computer/calculator.h b/src/VecSim/spaces/computer/calculator.h index a82293700..539325d8e 100644 --- a/src/VecSim/spaces/computer/calculator.h +++ b/src/VecSim/spaces/computer/calculator.h @@ -23,6 +23,9 @@ class IndexCalculatorInterface : public VecsimBaseObject { virtual ~IndexCalculatorInterface() = default; virtual DistType calcDistance(const void *v1, const void *v2, size_t dim) const = 0; + + // Raw distance function; cached by the index to skip the vtable on the hot path. + virtual spaces::dist_func_t getDistFunc() const = 0; }; /** @@ -56,4 +59,6 @@ class DistanceCalculatorCommon DistType calcDistance(const void *v1, const void *v2, size_t dim) const override { return this->dist_func(v1, v2, dim); } + + spaces::dist_func_t getDistFunc() const override { return this->dist_func; } }; diff --git a/src/VecSim/vec_sim_index.h b/src/VecSim/vec_sim_index.h index 88eabea69..e5a5183b7 100644 --- a/src/VecSim/vec_sim_index.h +++ b/src/VecSim/vec_sim_index.h @@ -86,6 +86,7 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { RawDataContainer *vectors; // The raw vectors data container. private: IndexCalculatorInterface *indexCalculator; // Distance calculator. + spaces::dist_func_t cachedDistFunc; // Cached dist func, used on the hot path. PreprocessorsContainerAbstract *preprocessors; // Storage and query preprocessors. size_t inputBlobSize; // The size of input vectors/queries blob in bytes. May differ from dim * @@ -124,8 +125,11 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { metric(params.metric), blockSize(params.blockSize ? params.blockSize : DEFAULT_BLOCK_SIZE), lastMode(EMPTY_MODE), isMulti(params.multi), isDisk(params.isDisk), logCallbackCtx(params.logCtx), - indexCalculator(components.indexCalculator), preprocessors(components.preprocessors), - inputBlobSize(params.inputBlobSize), storedDataSize(params.storedDataSize) { + indexCalculator(components.indexCalculator), + cachedDistFunc(components.indexCalculator ? components.indexCalculator->getDistFunc() + : nullptr), + preprocessors(components.preprocessors), inputBlobSize(params.inputBlobSize), + storedDataSize(params.storedDataSize) { assert(VecSimType_sizeof(vecType)); assert(storedDataSize); assert(inputBlobSize); @@ -146,10 +150,16 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { /** * @brief Calculate the distance between two vectors based on index parameters. * + * Uses the cached dist func to avoid the indexCalculator vtable on the hot path. + * + * @note Precondition: @c cachedDistFunc must be non-null. Subclasses that construct + * this index with a null @c indexCalculator (e.g. SVS, which uses its own + * internal distance kernels) must not call this method. + * * @return the distance between the vectors. */ DistType calcDistance(const void *vector_data1, const void *vector_data2) const { - return indexCalculator->calcDistance(vector_data1, vector_data2, this->dim); + return cachedDistFunc(vector_data1, vector_data2, this->dim); } /** diff --git a/tests/unit/test_components.cpp b/tests/unit/test_components.cpp index bd3c8d642..2f91d2411 100644 --- a/tests/unit/test_components.cpp +++ b/tests/unit/test_components.cpp @@ -33,6 +33,9 @@ class DistanceCalculatorDummy : public DistanceCalculatorInterfacedist_func(7); } + + // Dummy uses a non-standard dist func signature, so the standard slot is unavailable. + spaces::dist_func_t getDistFunc() const override { return nullptr; } }; } // namespace dummyCalcultor From 4f9d216585ea03a2a5edb288a0c770c59d5fe36b Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Tue, 23 Jun 2026 08:57:23 -0700 Subject: [PATCH 02/64] initial --- src/VecSim/algorithms/svs/svs.h | 40 +++---- src/VecSim/algorithms/svs/svs_tiered.h | 140 +++++++++++++++---------- src/VecSim/vec_sim_tiered_index.h | 15 +++ 3 files changed, 116 insertions(+), 79 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 8b03514f0..200fdd51a 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -263,11 +263,6 @@ class SVSIndex : public VecSimIndexAbstract, fl } int deleted_num = 0; - if constexpr (!isMulti) { - // SVS index does not support overriding vectors with the same label - // so we have to delete them first if needed - deleted_num = deleteVectorsImpl(labels, n); - } std::span ids(labels, n); auto processed_blob = this->preprocessForBatchStorage(vectors_data, n); @@ -279,6 +274,11 @@ class SVSIndex : public VecSimIndexAbstract, fl // SVS index instance cannot be empty, so we have to construct it at first rows impl_ = initImpl(points, ids); } else { + if constexpr (!isMulti) { + // SVS index does not support overriding vectors with the same label + // so we have to delete them first if needed + deleted_num = deleteVectorsImpl(labels, n); + } // Add new points to existing SVS index impl_->add_points(points, ids); } @@ -287,7 +287,7 @@ class SVSIndex : public VecSimIndexAbstract, fl } int deleteVectorImpl(const labelType label) { - if (indexLabelCount() == 0 || !impl_->has_id(label)) { + if (indexLabelCount() == 0) { return 0; } @@ -302,20 +302,7 @@ class SVSIndex : public VecSimIndexAbstract, fl return 0; } - // SVS fails if we try to delete non-existing entries - std::vector entries_to_delete; - entries_to_delete.reserve(n); - for (size_t i = 0; i < n; i++) { - if (impl_->has_id(labels[i])) { - entries_to_delete.push_back(labels[i]); - } - } - - if (entries_to_delete.size() == 0) { - return 0; - } - - const auto deleted_num = impl_->delete_entries(entries_to_delete); + const auto deleted_num = impl_->delete_entries(std::span{labels, n}); this->markIndexUpdate(deleted_num); return deleted_num; @@ -553,13 +540,16 @@ class SVSIndex : public VecSimIndexAbstract, fl } double getDistanceFrom_Unsafe(labelType label, const void *vector_data) const override { - if (!impl_ || !impl_->has_id(label)) { - return std::numeric_limits::quiet_NaN(); - }; + if (!impl_) return std::numeric_limits::quiet_NaN(); auto query_datum = std::span{static_cast(vector_data), this->dim}; - auto dist = impl_->get_distance(label, query_datum); - return toVecSimDistance(dist); + try { + auto dist = impl_->get_distance(label, query_datum); + return std::isnan(dist) ? std::numeric_limits::quiet_NaN() + : toVecSimDistance(static_cast(dist)); + } catch (const svs::lib::ANNException &) { + return std::numeric_limits::quiet_NaN(); + } } VecSimQueryReply *topKQuery(const void *queryBlob, size_t k, diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 9bce50f89..947db01a1 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -233,7 +233,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::atomic_flag indexGCScheduled = ATOMIC_FLAG_INIT; // Used to prevent running multiple index update jobs in parallel. // Even if update jobs scheduled sequentially, they can be started in parallel. - mutable std::mutex updateJobMutex; + // mutable std::shared_mutex updateJobMutex; // The reason of following container just to properly destroy jobs which not executed yet SVSMultiThreadJob::JobsRegistry uncompletedJobs; @@ -261,7 +261,6 @@ class TieredSVSIndex : public VecSimTieredIndex { VecSimBatchIterator *flat_iterator; VecSimBatchIterator *svs_iterator; - std::shared_lock svs_lock; // On single value indices, this set holds the IDs of the results that were returned from // the flat buffer. @@ -327,7 +326,6 @@ class TieredSVSIndex : public VecSimTieredIndex { void acquire_svs_iterator() { assert(svs_iterator == nullptr); - this->index->mainIndexGuard.lock_shared(); svs_iterator = index->backendIndex->newBatchIterator( this->flat_iterator->getQueryBlob(), queryParams); } @@ -336,7 +334,6 @@ class TieredSVSIndex : public VecSimTieredIndex { if (svs_iterator != nullptr && svs_iterator != depleted()) { delete svs_iterator; svs_iterator = nullptr; - this->index->mainIndexGuard.unlock_shared(); } } @@ -361,7 +358,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::move(allocator)), index(index), flat_results(this->allocator), svs_results(this->allocator), flat_iterator(index->frontendIndex->newBatchIterator(query_vector, queryParams)), - svs_iterator(nullptr), svs_lock(index->mainIndexGuard, std::defer_lock), + svs_iterator(nullptr), returned_results_set(this->allocator) { if (queryParams) { this->queryParams = @@ -548,7 +545,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto index = static_cast *>(idx); assert(index); // prevent parallel updates - std::lock_guard lock(index->updateJobMutex); + std::lock_guard lock(index->updateJobMutex); // Release the scheduled flag to allow scheduling again index->indexUpdateScheduled.clear(); // Update the SVS index @@ -575,7 +572,6 @@ class TieredSVSIndex : public VecSimTieredIndex { auto index = static_cast *>(idx); assert(index); - std::lock_guard lock{index->mainIndexGuard}; // Release the scheduled flag to allow scheduling again index->indexGCScheduled.clear(); @@ -605,7 +601,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto jobs = SVSMultiThreadJob::createJobs( this->allocator, SVS_BATCH_UPDATE_JOB, updateSVSIndexWrapper, this, total_threads, std::chrono::microseconds(updateJobWaitTime), &uncompletedJobs); - this->submitJobs(jobs); + this->submitUpdateJobs(jobs); } void scheduleSVSIndexGC() { @@ -678,29 +674,24 @@ class TieredSVSIndex : public VecSimTieredIndex { executeTracingCallback("UpdateJob::before_add_to_svs"); { // lock backend index for writing and add vectors there - std::shared_lock main_shared_lock(this->mainIndexGuard); auto svs_index = GetSVSIndex(); assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); if (this->backendIndex->indexSize() == 0) { - // If backend index is empty, we need to initialize it first. svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); - // Upgrade to unique lock to set the new impl - main_shared_lock.unlock(); - std::lock_guard lock(this->mainIndexGuard); + // std::lock_guard lock(this->mainIndexGuard); svs_index->setImpl(std::move(impl)); } else { - // Backend index is initialized - just add the vectors. - main_shared_lock.unlock(); - std::lock_guard lock(this->mainIndexGuard); - // Upgrade to unique lock to add vectors + // std::lock_guard lock(this->mainIndexGuard); + // std::shared_lock lock(this->mainIndexGuard); svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); } } + executeTracingCallback("UpdateJob::after_add_to_svs"); // clean-up frontend index { // lock frontend index for writing and delete moved vectors @@ -732,8 +723,6 @@ class TieredSVSIndex : public VecSimTieredIndex { // delete vectors from backend index that were deleted from the frontend index during // the update process. { - std::lock_guard main_lock(this->mainIndexGuard); - std::sort(deleted_labels_during_update.begin(), deleted_labels_during_update.end()); auto it = std::unique(deleted_labels_during_update.begin(), deleted_labels_during_update.end()); @@ -791,7 +780,6 @@ class TieredSVSIndex : public VecSimTieredIndex { // It is ok to lock everything at once for in-place mode, // but we will have to unlock averything before calling updateSVSIndexWrapper() // so make the minimal needed lock here. - std::shared_lock backend_shared_lock(this->mainIndexGuard); // Backend index initialization data have to be buffered for proper // compression/training. if (this->backendIndex->indexSize() == 0) { @@ -807,18 +795,18 @@ class TieredSVSIndex : public VecSimTieredIndex { // ... move vectors to the backend index. if (frontend_index_size >= this->trainingTriggerThreshold) { // updateSVSIndexWrapper() accures it's own locks - backend_shared_lock.unlock(); + // backend_shared_lock.unlock(); // initialize the SVS index synchonously using current thread only updateSVSIndexWrapper(this, 1); } return ret; } else { // backend index is initialized - we can add the vector directly - backend_shared_lock.unlock(); + // backend_shared_lock.unlock(); auto storage_blob = this->frontendIndex->preprocessForStorage(blob); // prevent update job from running in parallel and lock any access to the backend // index - std::scoped_lock lock(this->updateJobMutex, this->mainIndexGuard); + std::lock_guard lock(this->updateJobMutex); // Set available thread count to 1 for single vector write-in-place operation. // This maintains the contract that single vector operations use exactly one thread. // TODO: Replace this setNumThreads(1) call with an assertion once we establish @@ -840,15 +828,7 @@ class TieredSVSIndex : public VecSimTieredIndex { } } // Remove vector from the backend index if it exists in case of non-MULTI. - auto label_exists = [&]() { - std::shared_lock lock(this->mainIndexGuard); - return svs_index->isLabelExists(label); - }(); - - if (label_exists) { - std::lock_guard lock(this->mainIndexGuard); - ret -= this->backendIndex->deleteVector(label); - } + ret -= this->backendIndex->deleteVector(label); } { // Add vector to the frontend index. std::lock_guard lock(this->flatIndexGuard); @@ -869,7 +849,6 @@ class TieredSVSIndex : public VecSimTieredIndex { { // If main index is empty then update_threshold is trainingTriggerThreshold, // overwise it is updateTriggerThreshold. - std::shared_lock lock(this->mainIndexGuard); update_threshold = this->backendIndex->indexSize() == 0 ? this->trainingTriggerThreshold : this->updateTriggerThreshold; } @@ -936,15 +915,7 @@ class TieredSVSIndex : public VecSimTieredIndex { ret = this->deleteAndRecordSwaps_Unsafe(label); } - label_exists = [&]() { - std::shared_lock lock(this->mainIndexGuard); - return svs_index->isLabelExists(label); - }(); - - if (label_exists) { - std::lock_guard lock(this->mainIndexGuard); - ret += this->backendIndex->deleteVector(label); - } + ret += this->backendIndex->deleteVector(label); return ret; } size_t getNumMarkedDeleted() const override { @@ -953,13 +924,11 @@ class TieredSVSIndex : public VecSimTieredIndex { size_t indexSize() const override { std::shared_lock flat_lock(this->flatIndexGuard); - std::shared_lock main_lock(this->mainIndexGuard); return this->frontendIndex->indexSize() + this->backendIndex->indexSize(); } size_t indexCapacity() const override { std::shared_lock flat_lock(this->flatIndexGuard); - std::shared_lock main_lock(this->mainIndexGuard); return this->frontendIndex->indexCapacity() + this->backendIndex->indexCapacity(); } @@ -997,7 +966,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // the entire training duration (which can take 40-85s on slow machines). // If the mutex is held, training is actively running, so we report // indexUpdateScheduled = true (BACKGROUND_INDEXING = 1). - std::unique_lock lock(this->updateJobMutex, std::try_to_lock); + std::unique_lock lock(this->updateJobMutex, std::try_to_lock); if (lock.owns_lock()) { svsTieredInfo.indexUpdateScheduled = this->indexUpdateScheduled.test() == VecSimBool_TRUE; @@ -1052,17 +1021,81 @@ class TieredSVSIndex : public VecSimTieredIndex { VecSimQueryReply *topKQuery(const void *queryBlob, size_t k, VecSimQueryParams *queryParams) const override { - // SVS implements it's own distance computation functions which may cause sligthly different - // distance values than VecSim Flat Index does, so we always have to merge results with set. - return this->template topKQueryImp(queryBlob, k, queryParams); + // SVS handles its own internal locking for concurrent search + modification, + // so we don't need mainIndexGuard for backend queries. + this->flatIndexGuard.lock_shared(); + + if (this->frontendIndex->indexSize() == 0) { + this->flatIndexGuard.unlock_shared(); + + auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); + const void *processed_query = processed_query_ptr.get(); + return this->backendIndex->topKQuery(processed_query, k, queryParams); + } else { + auto flat_results = this->frontendIndex->topKQuery(queryBlob, k, queryParams); + this->flatIndexGuard.unlock_shared(); + + if (flat_results->code != VecSim_QueryReply_OK) { + assert(flat_results->results.empty()); + return flat_results; + } + + auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); + const void *processed_query = processed_query_ptr.get(); + auto main_results = this->backendIndex->topKQuery(processed_query, k, queryParams); + + if (main_results->code != VecSim_QueryReply_OK) { + VecSimQueryReply_Free(flat_results); + assert(main_results->results.empty()); + return main_results; + } + + return merge_result_lists(main_results, flat_results, k); + } } VecSimQueryReply *rangeQuery(const void *queryBlob, double radius, VecSimQueryParams *queryParams, VecSimQueryReply_Order order) const override { - // SVS implements it's own distance computation functions which may cause sligthly different - // distance values than VecSim Flat Index does, so we always have to merge results with set. - return this->template rangeQueryImp(queryBlob, radius, queryParams, order); + // SVS handles its own internal locking for concurrent search + modification, + // so we don't need mainIndexGuard for backend queries. + this->flatIndexGuard.lock_shared(); + + if (this->frontendIndex->indexSize() == 0) { + this->flatIndexGuard.unlock_shared(); + + auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); + const void *processed_query = processed_query_ptr.get(); + auto res = this->backendIndex->rangeQuery(processed_query, radius, queryParams); + sort_results(res, order); + return res; + } else { + auto flat_results = this->frontendIndex->rangeQuery(queryBlob, radius, queryParams); + this->flatIndexGuard.unlock_shared(); + + if (flat_results->code != VecSim_QueryReply_OK) { + return flat_results; + } + + auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); + const void *processed_query = processed_query_ptr.get(); + auto main_results = + this->backendIndex->rangeQuery(processed_query, radius, queryParams); + + if (BY_SCORE == order) { + sort_results_by_score_then_id(main_results); + sort_results_by_score_then_id(flat_results); + + auto code = main_results->code; + VecSimQueryReply *ret = merge_result_lists(main_results, flat_results, -1); + ret->code = code; + return ret; + } else { // BY_ID + concat_results(main_results, flat_results); + filter_results_by_id(main_results); + return main_results; + } + } } VecSimBatchIterator *newBatchIterator(const void *queryBlob, @@ -1081,7 +1114,6 @@ class TieredSVSIndex : public VecSimTieredIndex { TIERED_LOG(VecSimCommonStrings::LOG_VERBOSE_STRING, "running synchronous GC for tiered SVS index in write-in-place mode"); // In write-in-place mode, we run GC synchronously. - std::lock_guard lock{this->mainIndexGuard}; if (this->backendIndex->indexSize() == 0) { // No need to run GC on an empty index. return; @@ -1099,11 +1131,11 @@ class TieredSVSIndex : public VecSimTieredIndex { void acquireSharedLocks() override { this->flatIndexGuard.lock_shared(); - this->mainIndexGuard.lock_shared(); + // this->mainIndexGuard.lock_shared(); } void releaseSharedLocks() override { - this->mainIndexGuard.unlock_shared(); + // this->mainIndexGuard.unlock_shared(); this->flatIndexGuard.unlock_shared(); } }; diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index d097b1ae4..4f8b109a9 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -50,6 +50,7 @@ class VecSimTieredIndex : public VecSimIndexInterface { mutable std::shared_mutex flatIndexGuard; mutable std::shared_mutex mainIndexGuard; + mutable std::shared_mutex updateJobMutex; void lockMainIndexGuard() const { mainIndexGuard.lock(); #ifdef BUILD_TESTS @@ -76,6 +77,16 @@ class VecSimTieredIndex : public VecSimIndexInterface { jobs.size()); } + void submitUpdateJobs(vecsim_stl::vector &jobs) { + vecsim_stl::vector callbacks(jobs.size(), this->allocator); + for (size_t i = 0; i < jobs.size(); i++) { + callbacks[i] = jobs[i]->Execute; + } + std::shared_lock lock(this->updateJobMutex); + this->SubmitJobsToQueue(this->jobQueue, this->jobQueueCtx, jobs.data(), callbacks.data(), + jobs.size()); + } + /** * @brief Return the union of unique labels in both index tiers (which are not deleted). * This is a debug-only method for tiered indexes that computes the union of labels @@ -181,6 +192,7 @@ VecSimTieredIndex::topKQueryImp(const void *queryBlob, size_ // Simply query the main index and return the results while holding the lock. auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); + assert(false); this->mainIndexGuard.lock_shared(); auto res = this->backendIndex->topKQuery(processed_query, k, queryParams); this->mainIndexGuard.unlock_shared(); @@ -201,6 +213,7 @@ VecSimTieredIndex::topKQueryImp(const void *queryBlob, size_ auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); // Lock the main index and query it. + assert(false); this->mainIndexGuard.lock_shared(); auto main_results = this->backendIndex->topKQuery(processed_query, k, queryParams); this->mainIndexGuard.unlock_shared(); @@ -263,6 +276,7 @@ VecSimTieredIndex::rangeQueryImp(const void *queryBlob, doub auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); // Simply query the main index and return the results while holding the lock. + assert(false); this->mainIndexGuard.lock_shared(); auto res = this->backendIndex->rangeQuery(processed_query, radius, queryParams); this->mainIndexGuard.unlock_shared(); @@ -286,6 +300,7 @@ VecSimTieredIndex::rangeQueryImp(const void *queryBlob, doub auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); // Lock the main index and query it. + assert(false); this->mainIndexGuard.lock_shared(); auto main_results = this->backendIndex->rangeQuery(processed_query, radius, queryParams); this->mainIndexGuard.unlock_shared(); From b5c1c0831cdbf20476ca6280a87d7cb624332ea1 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Fri, 26 Jun 2026 04:17:42 -0700 Subject: [PATCH 03/64] optimization --- src/VecSim/algorithms/svs/svs.h | 15 +- src/VecSim/algorithms/svs/svs_tiered.h | 244 ++++++++++++++++++++++--- src/VecSim/vec_sim_common.h | 1 + src/VecSim/vec_sim_tiered_index.h | 15 -- 4 files changed, 229 insertions(+), 46 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 200fdd51a..9043671a2 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -37,6 +37,7 @@ struct SVSIndexBase { SVSIndexBase() : num_marked_deleted{0} {}; virtual ~SVSIndexBase() = default; + virtual int addVector(const void *vector_data, labelType label) = 0; virtual int addVectors(const void *vectors_data, const labelType *labels, size_t n) = 0; virtual int deleteVectors(const labelType *labels, size_t n) = 0; virtual bool isLabelExists(labelType label) const = 0; @@ -45,7 +46,9 @@ struct SVSIndexBase virtual void setNumThreads(size_t numThreads) = 0; virtual size_t getThreadPoolCapacity() const = 0; virtual bool isCompressed() const = 0; - size_t getNumMarkedDeleted() const { return num_marked_deleted; } + size_t getNumMarkedDeleted() const { + return num_marked_deleted.load(std::memory_order_relaxed); + } // Abstract handler to manage SVS implementation instance // declared to avoid unsafe unique_ptr usage @@ -62,7 +65,7 @@ struct SVSIndexBase protected: // Index marked deleted vectors counter to initiate reindexing if it exceeds threshold // markIndexUpdate() manages this counter - size_t num_marked_deleted; + std::atomic num_marked_deleted; }; /** Thread Management Strategy: @@ -316,11 +319,11 @@ class SVSIndex : public VecSimIndexAbstract, fl // SVS index instance should not be empty if (indexLabelCount() == 0) { this->impl_.reset(); - num_marked_deleted = 0; + num_marked_deleted.store(0, std::memory_order_relaxed); return; } - num_marked_deleted += n; + num_marked_deleted.fetch_add(n, std::memory_order_relaxed); } bool isTwoLevelLVQ(const VecSimSvsQuantBits &qbits) { @@ -506,7 +509,7 @@ class SVSIndex : public VecSimIndexAbstract, fl // Enforce single-threaded execution for single vector operations to ensure optimal // performance and consistent behavior. Callers must set numThreads=1 before calling this // method. - assert(getNumThreads() == 1 && "Can't use more than one thread to insert a single vector"); + // assert(getNumThreads() == 1 && "Can't use more than one thread to insert a single vector"); return addVectorsImpl(vector_data, &label, 1); } @@ -721,7 +724,7 @@ class SVSIndex : public VecSimIndexAbstract, fl // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.compact impl_->compact(); } - num_marked_deleted = 0; + num_marked_deleted.store(0, std::memory_order_relaxed); } #ifdef BUILD_TESTS diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 947db01a1..82fd33b2c 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -13,6 +13,19 @@ #include #include +/** + * Definition of a job that inserts a new vector from flat into SVS Index. + */ +struct SVSInsertJob : public AsyncJob { + labelType label; + idType id; + + SVSInsertJob(std::shared_ptr allocator, labelType label_, idType id_, + JobCallback insertCb, VecSimIndex *index_) + : AsyncJob(allocator, SVS_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} +}; + + /** * @class SVSMultiThreadJob * @brief Represents a multi-threaded asynchronous job for the SVS algorithm. @@ -200,10 +213,11 @@ class SVSMultiThreadJob : public AsyncJob { template class TieredSVSIndex : public VecSimTieredIndex { + using DistType = float; using Self = TieredSVSIndex; - using Base = VecSimTieredIndex; - using flat_index_t = BruteForceIndex; - using backend_index_t = VecSimIndexAbstract; + using Base = VecSimTieredIndex; + using flat_index_t = BruteForceIndex; + using backend_index_t = VecSimIndexAbstract; using svs_index_t = SVSIndexBase; // swaps_journal is used by updateSVSIndex() to track vectors swap operations that were done in @@ -233,11 +247,22 @@ class TieredSVSIndex : public VecSimTieredIndex { std::atomic_flag indexGCScheduled = ATOMIC_FLAG_INIT; // Used to prevent running multiple index update jobs in parallel. // Even if update jobs scheduled sequentially, they can be started in parallel. - // mutable std::shared_mutex updateJobMutex; + mutable std::shared_mutex updateJobMutex; // The reason of following container just to properly destroy jobs which not executed yet SVSMultiThreadJob::JobsRegistry uncompletedJobs; + std::atomic backendReady{false}; + std::atomic backendInitSubmited{false}; + + vecsim_stl::unordered_map> labelToInsertJobs; + // A mapping to hold invalid jobs, so we can dispose them upon index deletion. + vecsim_stl::unordered_map invalidJobs; + idType currInvalidJobId; // A unique arbitrary identifier for accessing invalid jobs + std::mutex invalidJobsLookupGuard; + + size_t flat_buffer_bound; + /// //////////////////////////////////////////////////////////////////////////////////////////////////// // TieredSVS_BatchIterator // @@ -546,10 +571,17 @@ class TieredSVSIndex : public VecSimTieredIndex { assert(index); // prevent parallel updates std::lock_guard lock(index->updateJobMutex); - // Release the scheduled flag to allow scheduling again - index->indexUpdateScheduled.clear(); // Update the SVS index index->updateSVSIndex(availableThreads); + // Release the scheduled flag to allow scheduling again + index->indexUpdateScheduled.clear(); + } + + static void executeInsertJobWrapper(AsyncJob *job) { + auto *insert_job = static_cast(job); + auto *job_index = static_cast *>(insert_job->index); + job_index->executeInsertJob(insert_job); + delete job; } /** @@ -591,6 +623,21 @@ class TieredSVSIndex : public VecSimTieredIndex { #ifdef BUILD_TESTS public: #endif + + void updateInsertJobInternalId(idType prev_id, idType new_id, labelType label) { + // Update the pending job id, due to a swap that was caused after the removal of new_id. + assert(new_id != INVALID_ID && prev_id != INVALID_ID); + auto it = this->labelToInsertJobs.find(label); + if (it != this->labelToInsertJobs.end()) { + // There is a pending job for the label of the swapped last id - update its id. + for (SVSInsertJob *job_it : it->second) { + if (job_it->id == prev_id) { + job_it->id = new_id; + } + } + } + } + void scheduleSVSIndexUpdate() { // do not schedule if scheduled already if (indexUpdateScheduled.test_and_set()) { @@ -601,7 +648,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto jobs = SVSMultiThreadJob::createJobs( this->allocator, SVS_BATCH_UPDATE_JOB, updateSVSIndexWrapper, this, total_threads, std::chrono::microseconds(updateJobWaitTime), &uncompletedJobs); - this->submitUpdateJobs(jobs); + this->submitJobs(jobs); } void scheduleSVSIndexGC() { @@ -648,6 +695,73 @@ class TieredSVSIndex : public VecSimTieredIndex { } } + idType setAndSaveInvalidJob(AsyncJob *job) { + this->invalidJobsLookupGuard.lock(); + job->isValid = false; + idType curInvalidId = currInvalidJobId++; + this->invalidJobs.insert({curInvalidId, job}); + this->invalidJobsLookupGuard.unlock(); + return curInvalidId; + } + + void executeInsertJob(SVSInsertJob *job) { + assert(this->backendReady.load(std::memory_order_acquire)); + + // Note that accessing the job fields should occur with flat index guard held (here and later). + this->flatIndexGuard.lock_shared(); + if (!job->isValid) { + this->flatIndexGuard.unlock_shared(); + // Job has been invalidated in the meantime - nothing to execute, and remove it from the + // lookup. + this->invalidJobsLookupGuard.lock(); + this->invalidJobs.erase(job->id); + this->invalidJobsLookupGuard.unlock(); + return; + } + this->flatIndexGuard.unlock_shared(); + + auto svs_index = GetSVSIndex(); + auto storage_blob = this->frontendIndex->preprocessForStorage(this->frontendIndex->getDataByInternalId(job->id)); + svs_index->addVector(storage_blob.get(), job->label); + + // Remove the vector and the insert job from the flat buffer. + this->flatIndexGuard.lock(); + // The job might have been invalidated due to overwrite in the meantime. In this case, + // it was already deleted and the job has been evicted. Otherwise, we need to do it now. + if (job->isValid) { + // Remove the job pointer from the labelToInsertJobs mapping. + auto &jobs = labelToInsertJobs.at(job->label); + for (size_t i = 0; i < jobs.size(); i++) { + if (jobs[i]->id == job->id) { + jobs.erase(jobs.begin() + (long)i); + break; + } + } + if (labelToInsertJobs.at(job->label).empty()) { + labelToInsertJobs.erase(job->label); + } + // Remove the vector from the flat buffer. + // The flat buffer stores data in a contiguous + // array, so deleting an element may move the last element into the freed slot to keep + // ids dense. Capture the last id's label beforehand (after deletion it is no longer in + // the lookup) so we can later fix up its insert job. + labelType last_vec_label = + this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); + int deleted = this->frontendIndex->deleteVectorById(job->label, job->id); + if (deleted && job->id != this->frontendIndex->indexSize()) { + // If the vector removal caused a swap with the last id, update the relevant insert job. + this->updateInsertJobInternalId(this->frontendIndex->indexSize(), job->id, + last_vec_label); + } + } else { + // Remove the current job from the invalid jobs' lookup, as we are about to delete it now. + this->invalidJobsLookupGuard.lock(); + this->invalidJobs.erase(job->id); + this->invalidJobsLookupGuard.unlock(); + } + this->flatIndexGuard.unlock(); + } + void updateSVSIndex(size_t availableThreads) { std::vector labels_to_move; std::vector vectors_to_move; @@ -658,6 +772,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto flat_index = this->GetFlatIndex(); const auto frontend_index_size = this->frontendIndex->indexSize(); + // fprintf(stderr, "updateSVSIndex: frontend_index_size = %ld\n", frontend_index_size); const size_t dim = flat_index->getDim(); labels_to_move.reserve(frontend_index_size); vectors_to_move.reserve(frontend_index_size * dim); @@ -681,12 +796,9 @@ class TieredSVSIndex : public VecSimTieredIndex { auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); - // std::lock_guard lock(this->mainIndexGuard); svs_index->setImpl(std::move(impl)); } else { - // std::lock_guard lock(this->mainIndexGuard); - // std::shared_lock lock(this->mainIndexGuard); - svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); + // svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); } @@ -709,6 +821,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // improvement int deleted = 0; idType id = labels_to_move.size(); + // fprintf(stderr, "updateSVSIndexMidle: frontend_index_size = %ld\n", this->frontendIndex->indexSize()); while (id-- > 0) { auto label = labels_to_move[id]; // Delete the vector from the frontend index if not in-place updated. @@ -716,6 +829,7 @@ class TieredSVSIndex : public VecSimTieredIndex { deleted += this->frontendIndex->deleteVectorById(label, id); } } + // fprintf(stderr, "updateSVSIndexEnd: frontend_index_size = %ld\n", this->frontendIndex->indexSize()); assert(deleted == std::count_if(labels_to_move.begin(), labels_to_move.end(), [](labelType label) { return label != SKIP_LABEL; }) && "Deleted vectors count does not match the number of labels to delete"); @@ -731,6 +845,7 @@ class TieredSVSIndex : public VecSimTieredIndex { svs_index->deleteVectors(deleted_labels_during_update.data(), deleted_labels_during_update.size()); } + this->backendReady.store(true, std::memory_order_release); } public: @@ -738,11 +853,14 @@ class TieredSVSIndex : public VecSimTieredIndex { const TieredIndexParams &tiered_index_params, std::shared_ptr allocator) : Base(svs_index, bf_index, tiered_index_params, allocator), - uncompletedJobs(this->allocator) { + uncompletedJobs(this->allocator), + labelToInsertJobs(this->allocator), + invalidJobs(this->allocator), + currInvalidJobId(0) { const auto &tiered_svs_params = tiered_index_params.specificParams.tieredSVSParams; // If flatBufferLimit is not initialized (0), use the default update threshold. - const size_t flat_buffer_bound = tiered_index_params.flatBufferLimit == 0 + flat_buffer_bound = tiered_index_params.flatBufferLimit == 0 ? SVS_VAMANA_DEFAULT_UPDATE_THRESHOLD : tiered_index_params.flatBufferLimit; @@ -830,7 +948,9 @@ class TieredSVSIndex : public VecSimTieredIndex { // Remove vector from the backend index if it exists in case of non-MULTI. ret -= this->backendIndex->deleteVector(label); } - { // Add vector to the frontend index. + + if (!this->backendInitSubmited.load(std::memory_order_acquire)) { + // Add vector to the frontend index. std::lock_guard lock(this->flatIndexGuard); const auto ft_ret = this->frontendIndex->addVector(blob, label); @@ -843,20 +963,94 @@ class TieredSVSIndex : public VecSimTieredIndex { deleted_labels_journal.push_back(label); } ret = std::max(ret + ft_ret, 0); - // Check frontend index size to determine if an update job schedule is needed. - frontend_index_size = this->frontendIndex->indexSize(); - } - { - // If main index is empty then update_threshold is trainingTriggerThreshold, - // overwise it is updateTriggerThreshold. - update_threshold = this->backendIndex->indexSize() == 0 ? this->trainingTriggerThreshold - : this->updateTriggerThreshold; + + if (this->frontendIndex->indexSize() >= this->trainingTriggerThreshold) { + this->backendInitSubmited.store(true, std::memory_order_release); + scheduleSVSIndexUpdate(); + // fprintf(stderr, "Init submited: this->frontendIndex->indexSize() = %ld\n", this->frontendIndex->indexSize()); + } + return ret; } - if (frontend_index_size >= update_threshold) { - scheduleSVSIndexUpdate(); + + // fprintf(stderr, "Waiting for backend ready\t"); + // spin-lock, while backend is initilizing + while (!this->backendReady.load(std::memory_order_acquire)) { + __builtin_ia32_pause(); } + // fprintf(stderr, "done: this->frontendIndex->indexSize() = %ld\n", this->frontendIndex->indexSize()); - return ret; + if (this->frontendIndex->indexSize() >= flat_buffer_bound) { + auto storage_blob = this->frontendIndex->preprocessForStorage(blob); + return ret = svs_index->addVector(storage_blob.get(), label); + } else { + this->flatIndexGuard.lock(); + idType new_flat_id = this->frontendIndex->indexSize(); + if (this->frontendIndex->isLabelExists(label) && !this->frontendIndex->isMultiValue()) { + // Overwrite the vector and invalidate its only pending job (since we are not in MULTI). + auto *old_job = this->labelToInsertJobs.at(label).at(0); + old_job->id = this->setAndSaveInvalidJob(old_job); + this->labelToInsertJobs.erase(label); + ret = 0; + // We are going to update the internal id that currently holds the vector associated with + // the given label. + new_flat_id = + dynamic_cast *>(this->frontendIndex) + ->getIdOfLabel(label); + // If we are adding a new element (rather than updating an exiting one) we may need to + // increase index capacity. + } + // If this label already exists, this will do overwrite. + this->frontendIndex->addVector(blob, label); + + AsyncJob *new_insert_job = new (this->allocator) + SVSInsertJob(this->allocator, label, new_flat_id, executeInsertJobWrapper, this); + // Save a pointer to the job, so that if the vector is overwritten, we'll have an indication. + if (this->labelToInsertJobs.find(label) != this->labelToInsertJobs.end()) { + // There's already a pending insert job for this label, add another one (without overwrite, + // only possible in multi index) + assert(this->backendIndex->isMultiValue()); + this->labelToInsertJobs.at(label).push_back((SVSInsertJob *)new_insert_job); + } else { + vecsim_stl::vector new_jobs_vec(1, (SVSInsertJob *)new_insert_job, + this->allocator); + this->labelToInsertJobs.insert({label, new_jobs_vec}); + } + this->flatIndexGuard.unlock(); + + // Insert job to the queue and signal the workers' updater. + this->submitSingleJob(new_insert_job); + return ret; + // } else { + // // // Add vector to the frontend index. + // // std::lock_guard lock(this->flatIndexGuard); + // // const auto ft_ret = this->frontendIndex->addVector(blob, label); + + // // if (ft_ret == 0) { // Vector was overriden - add 'skiping' swap to the journal. + // // assert(!this->backendIndex->isMultiValue() && + // // "addVector() may return 0 for single value indices only"); + // // for (auto id : this->frontendIndex->getElementIds(label)) { + // // this->swaps_journal.emplace_back(SKIP_LABEL, id, id); + // // } + // // deleted_labels_journal.push_back(label); + // // } + // // ret = std::max(ret + ft_ret, 0); + + // // // Check frontend index size to determine if an update job schedule is needed. + // // frontend_index_size = this->frontendIndex->indexSize(); + // // // if (this->backendInitSubmited.load(std::memory_order_acquire)) { + // // // if (frontend_index_size >= this->updateTriggerThreshold) { + // // // if (this->backendReady.load(std::memory_order_acquire)) { + // // // scheduleSVSIndexUpdate(); + // // // } + // // // } + // // // } else { + // // if (frontend_index_size >= this->trainingTriggerThreshold) { + // // this->backendInitSubmited.store(true, std::memory_order_release); + // // scheduleSVSIndexUpdate(); + // // } + // // // } + // // return ret; + } } int deleteAndRecordSwaps_Unsafe(labelType label) { diff --git a/src/VecSim/vec_sim_common.h b/src/VecSim/vec_sim_common.h index 82aceaa1c..03e7870f3 100644 --- a/src/VecSim/vec_sim_common.h +++ b/src/VecSim/vec_sim_common.h @@ -273,6 +273,7 @@ typedef enum { HNSW_SWAP_JOB, HNSW_DISK_JOB, SVS_BATCH_UPDATE_JOB, + SVS_INSERT_VECTOR_JOB, SVS_GC_JOB, INVALID_JOB // to indicate that finding a JobType >= INVALID_JOB is an error } JobType; diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 4f8b109a9..d097b1ae4 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -50,7 +50,6 @@ class VecSimTieredIndex : public VecSimIndexInterface { mutable std::shared_mutex flatIndexGuard; mutable std::shared_mutex mainIndexGuard; - mutable std::shared_mutex updateJobMutex; void lockMainIndexGuard() const { mainIndexGuard.lock(); #ifdef BUILD_TESTS @@ -77,16 +76,6 @@ class VecSimTieredIndex : public VecSimIndexInterface { jobs.size()); } - void submitUpdateJobs(vecsim_stl::vector &jobs) { - vecsim_stl::vector callbacks(jobs.size(), this->allocator); - for (size_t i = 0; i < jobs.size(); i++) { - callbacks[i] = jobs[i]->Execute; - } - std::shared_lock lock(this->updateJobMutex); - this->SubmitJobsToQueue(this->jobQueue, this->jobQueueCtx, jobs.data(), callbacks.data(), - jobs.size()); - } - /** * @brief Return the union of unique labels in both index tiers (which are not deleted). * This is a debug-only method for tiered indexes that computes the union of labels @@ -192,7 +181,6 @@ VecSimTieredIndex::topKQueryImp(const void *queryBlob, size_ // Simply query the main index and return the results while holding the lock. auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); - assert(false); this->mainIndexGuard.lock_shared(); auto res = this->backendIndex->topKQuery(processed_query, k, queryParams); this->mainIndexGuard.unlock_shared(); @@ -213,7 +201,6 @@ VecSimTieredIndex::topKQueryImp(const void *queryBlob, size_ auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); // Lock the main index and query it. - assert(false); this->mainIndexGuard.lock_shared(); auto main_results = this->backendIndex->topKQuery(processed_query, k, queryParams); this->mainIndexGuard.unlock_shared(); @@ -276,7 +263,6 @@ VecSimTieredIndex::rangeQueryImp(const void *queryBlob, doub auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); // Simply query the main index and return the results while holding the lock. - assert(false); this->mainIndexGuard.lock_shared(); auto res = this->backendIndex->rangeQuery(processed_query, radius, queryParams); this->mainIndexGuard.unlock_shared(); @@ -300,7 +286,6 @@ VecSimTieredIndex::rangeQueryImp(const void *queryBlob, doub auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); // Lock the main index and query it. - assert(false); this->mainIndexGuard.lock_shared(); auto main_results = this->backendIndex->rangeQuery(processed_query, radius, queryParams); this->mainIndexGuard.unlock_shared(); From 417b970a6304df927cd60310a08cabf7227e9c59 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Fri, 26 Jun 2026 06:06:39 -0700 Subject: [PATCH 04/64] fix threadpool --- src/VecSim/algorithms/svs/svs_tiered.h | 34 +++----------------------- 1 file changed, 3 insertions(+), 31 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 82fd33b2c..2657679a4 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -797,10 +797,12 @@ class TieredSVSIndex : public VecSimTieredIndex { labels_to_move.size()); svs_index->setImpl(std::move(impl)); + svs_index->setNumThreads(1); } else { - // svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); + svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); + svs_index->setNumThreads(1); } } @@ -1020,36 +1022,6 @@ class TieredSVSIndex : public VecSimTieredIndex { // Insert job to the queue and signal the workers' updater. this->submitSingleJob(new_insert_job); return ret; - // } else { - // // // Add vector to the frontend index. - // // std::lock_guard lock(this->flatIndexGuard); - // // const auto ft_ret = this->frontendIndex->addVector(blob, label); - - // // if (ft_ret == 0) { // Vector was overriden - add 'skiping' swap to the journal. - // // assert(!this->backendIndex->isMultiValue() && - // // "addVector() may return 0 for single value indices only"); - // // for (auto id : this->frontendIndex->getElementIds(label)) { - // // this->swaps_journal.emplace_back(SKIP_LABEL, id, id); - // // } - // // deleted_labels_journal.push_back(label); - // // } - // // ret = std::max(ret + ft_ret, 0); - - // // // Check frontend index size to determine if an update job schedule is needed. - // // frontend_index_size = this->frontendIndex->indexSize(); - // // // if (this->backendInitSubmited.load(std::memory_order_acquire)) { - // // // if (frontend_index_size >= this->updateTriggerThreshold) { - // // // if (this->backendReady.load(std::memory_order_acquire)) { - // // // scheduleSVSIndexUpdate(); - // // // } - // // // } - // // // } else { - // // if (frontend_index_size >= this->trainingTriggerThreshold) { - // // this->backendInitSubmited.store(true, std::memory_order_release); - // // scheduleSVSIndexUpdate(); - // // } - // // // } - // // return ret; } } From 2d5d6999a1dbe990a2756546c8ccef7f109d6a7d Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Fri, 26 Jun 2026 08:42:23 -0700 Subject: [PATCH 05/64] minor fix --- src/VecSim/algorithms/svs/svs_tiered.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 2657679a4..48ea51dc8 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -718,10 +718,10 @@ class TieredSVSIndex : public VecSimTieredIndex { this->invalidJobsLookupGuard.unlock(); return; } + auto storage_blob = this->frontendIndex->preprocessForStorage(this->frontendIndex->getDataByInternalId(job->id)); this->flatIndexGuard.unlock_shared(); auto svs_index = GetSVSIndex(); - auto storage_blob = this->frontendIndex->preprocessForStorage(this->frontendIndex->getDataByInternalId(job->id)); svs_index->addVector(storage_blob.get(), job->label); // Remove the vector and the insert job from the flat buffer. @@ -1005,7 +1005,8 @@ class TieredSVSIndex : public VecSimTieredIndex { this->frontendIndex->addVector(blob, label); AsyncJob *new_insert_job = new (this->allocator) - SVSInsertJob(this->allocator, label, new_flat_id, executeInsertJobWrapper, this); + SVSInsertJob(this->allocator, label, new_flat_id, executeInsertJobWrapper, this); + // Save a pointer to the job, so that if the vector is overwritten, we'll have an indication. if (this->labelToInsertJobs.find(label) != this->labelToInsertJobs.end()) { // There's already a pending insert job for this label, add another one (without overwrite, From 3c3f842ab0dd8716f0b1e0aade43b17eaa3e55a0 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 29 Jun 2026 08:42:54 -0700 Subject: [PATCH 06/64] minor cleaning --- src/VecSim/algorithms/svs/svs.h | 4 ++-- src/VecSim/algorithms/svs/svs_tiered.h | 28 ++++++++------------------ 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 9043671a2..67e628d6d 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -351,7 +351,7 @@ class SVSIndex : public VecSimIndexAbstract, fl svs_details::getOrDefault(params.leanvec_dim, SVS_VAMANA_DEFAULT_LEANVEC_DIM)}, epsilon{svs_details::getOrDefault(params.epsilon, SVS_VAMANA_DEFAULT_EPSILON)}, is_two_level_lvq{isTwoLevelLVQ(params.quantBits)}, - threadpool_{std::max(size_t{SVS_VAMANA_DEFAULT_NUM_THREADS}, params.num_threads)}, + threadpool_{1}, impl_{nullptr} { logger_ = makeLogger(); } @@ -509,7 +509,7 @@ class SVSIndex : public VecSimIndexAbstract, fl // Enforce single-threaded execution for single vector operations to ensure optimal // performance and consistent behavior. Callers must set numThreads=1 before calling this // method. - // assert(getNumThreads() == 1 && "Can't use more than one thread to insert a single vector"); + assert(getNumThreads() == 1 && "Can't use more than one thread to insert a single vector"); return addVectorsImpl(vector_data, &label, 1); } diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 48ea51dc8..66c8ce530 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -792,17 +792,17 @@ class TieredSVSIndex : public VecSimTieredIndex { auto svs_index = GetSVSIndex(); assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); if (this->backendIndex->indexSize() == 0) { - svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); + // svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); svs_index->setImpl(std::move(impl)); - svs_index->setNumThreads(1); + // svs_index->setNumThreads(1); } else { - svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); + // svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); - svs_index->setNumThreads(1); + // svs_index->setNumThreads(1); } } @@ -938,19 +938,6 @@ class TieredSVSIndex : public VecSimTieredIndex { assert(this->getWriteMode() != VecSim_WriteInPlace && "InPlace mode returns early"); // Async mode - add vector to the frontend index and schedule an update job if needed. - if (!this->backendIndex->isMultiValue()) { - { - std::shared_lock lock(this->flatIndexGuard); - // If the label already exists in the frontend index, we should count it - // to prevent the case when existing vector is moved meanwhile by the update job. - if (this->frontendIndex->isLabelExists(label)) { - ret = -1; - } - } - // Remove vector from the backend index if it exists in case of non-MULTI. - ret -= this->backendIndex->deleteVector(label); - } - if (!this->backendInitSubmited.load(std::memory_order_acquire)) { // Add vector to the frontend index. std::lock_guard lock(this->flatIndexGuard); @@ -969,22 +956,23 @@ class TieredSVSIndex : public VecSimTieredIndex { if (this->frontendIndex->indexSize() >= this->trainingTriggerThreshold) { this->backendInitSubmited.store(true, std::memory_order_release); scheduleSVSIndexUpdate(); - // fprintf(stderr, "Init submited: this->frontendIndex->indexSize() = %ld\n", this->frontendIndex->indexSize()); } return ret; } - // fprintf(stderr, "Waiting for backend ready\t"); // spin-lock, while backend is initilizing while (!this->backendReady.load(std::memory_order_acquire)) { __builtin_ia32_pause(); } - // fprintf(stderr, "done: this->frontendIndex->indexSize() = %ld\n", this->frontendIndex->indexSize()); + this->flatIndexGuard.lock_shared(); if (this->frontendIndex->indexSize() >= flat_buffer_bound) { + this->flatIndexGuard.unlock_shared(); auto storage_blob = this->frontendIndex->preprocessForStorage(blob); + return ret = svs_index->addVector(storage_blob.get(), label); } else { + this->flatIndexGuard.unlock_shared(); this->flatIndexGuard.lock(); idType new_flat_id = this->frontendIndex->indexSize(); if (this->frontendIndex->isLabelExists(label) && !this->frontendIndex->isMultiValue()) { From 952efbf76ea788885a5e61b34a94cdb6d1004fae Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 1 Jul 2026 04:47:03 -0700 Subject: [PATCH 07/64] clean up --- src/VecSim/algorithms/svs/svs_tiered.h | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 66c8ce530..909da8266 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -772,7 +772,6 @@ class TieredSVSIndex : public VecSimTieredIndex { auto flat_index = this->GetFlatIndex(); const auto frontend_index_size = this->frontendIndex->indexSize(); - // fprintf(stderr, "updateSVSIndex: frontend_index_size = %ld\n", frontend_index_size); const size_t dim = flat_index->getDim(); labels_to_move.reserve(frontend_index_size); vectors_to_move.reserve(frontend_index_size * dim); @@ -792,17 +791,13 @@ class TieredSVSIndex : public VecSimTieredIndex { auto svs_index = GetSVSIndex(); assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); if (this->backendIndex->indexSize() == 0) { - // svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); svs_index->setImpl(std::move(impl)); - // svs_index->setNumThreads(1); } else { - // svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); - // svs_index->setNumThreads(1); } } @@ -823,7 +818,6 @@ class TieredSVSIndex : public VecSimTieredIndex { // improvement int deleted = 0; idType id = labels_to_move.size(); - // fprintf(stderr, "updateSVSIndexMidle: frontend_index_size = %ld\n", this->frontendIndex->indexSize()); while (id-- > 0) { auto label = labels_to_move[id]; // Delete the vector from the frontend index if not in-place updated. @@ -831,7 +825,6 @@ class TieredSVSIndex : public VecSimTieredIndex { deleted += this->frontendIndex->deleteVectorById(label, id); } } - // fprintf(stderr, "updateSVSIndexEnd: frontend_index_size = %ld\n", this->frontendIndex->indexSize()); assert(deleted == std::count_if(labels_to_move.begin(), labels_to_move.end(), [](labelType label) { return label != SKIP_LABEL; }) && "Deleted vectors count does not match the number of labels to delete"); @@ -915,14 +908,12 @@ class TieredSVSIndex : public VecSimTieredIndex { // ... move vectors to the backend index. if (frontend_index_size >= this->trainingTriggerThreshold) { // updateSVSIndexWrapper() accures it's own locks - // backend_shared_lock.unlock(); // initialize the SVS index synchonously using current thread only updateSVSIndexWrapper(this, 1); } return ret; } else { // backend index is initialized - we can add the vector directly - // backend_shared_lock.unlock(); auto storage_blob = this->frontendIndex->preprocessForStorage(blob); // prevent update job from running in parallel and lock any access to the backend // index @@ -1286,11 +1277,9 @@ class TieredSVSIndex : public VecSimTieredIndex { void acquireSharedLocks() override { this->flatIndexGuard.lock_shared(); - // this->mainIndexGuard.lock_shared(); } void releaseSharedLocks() override { - // this->mainIndexGuard.unlock_shared(); this->flatIndexGuard.unlock_shared(); } }; From 8265a7a15c43c932b080e702a8200de0adf1b2e7 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 1 Jul 2026 06:25:52 -0700 Subject: [PATCH 08/64] remove dead code --- deps/ScalableVectorSearch | 2 +- src/VecSim/algorithms/svs/svs.h | 13 +++++-------- src/VecSim/algorithms/svs/svs_tiered.h | 14 ++++++-------- 3 files changed, 12 insertions(+), 17 deletions(-) mode change 160000 => 120000 deps/ScalableVectorSearch diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch deleted file mode 160000 index 02dea9d9c..000000000 --- a/deps/ScalableVectorSearch +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 02dea9d9cd1c73dc03dc0fd91d14a548f1aece70 diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch new file mode 120000 index 000000000..25f58fd1a --- /dev/null +++ b/deps/ScalableVectorSearch @@ -0,0 +1 @@ +/home/drazdobu/libraries.ai.vector-search.svs \ No newline at end of file diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 67e628d6d..a3f2ca8c5 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -543,16 +543,13 @@ class SVSIndex : public VecSimIndexAbstract, fl } double getDistanceFrom_Unsafe(labelType label, const void *vector_data) const override { - if (!impl_) return std::numeric_limits::quiet_NaN(); + if (!impl_ || !impl_->has_id(label)) { + return std::numeric_limits::quiet_NaN(); + }; auto query_datum = std::span{static_cast(vector_data), this->dim}; - try { - auto dist = impl_->get_distance(label, query_datum); - return std::isnan(dist) ? std::numeric_limits::quiet_NaN() - : toVecSimDistance(static_cast(dist)); - } catch (const svs::lib::ANNException &) { - return std::numeric_limits::quiet_NaN(); - } + auto dist = impl_->get_distance(label, query_datum); + return toVecSimDistance(dist); } VecSimQueryReply *topKQuery(const void *queryBlob, size_t k, diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 909da8266..383734248 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -247,7 +247,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::atomic_flag indexGCScheduled = ATOMIC_FLAG_INIT; // Used to prevent running multiple index update jobs in parallel. // Even if update jobs scheduled sequentially, they can be started in parallel. - mutable std::shared_mutex updateJobMutex; + mutable std::mutex updateJobMutex; // The reason of following container just to properly destroy jobs which not executed yet SVSMultiThreadJob::JobsRegistry uncompletedJobs; @@ -570,11 +570,11 @@ class TieredSVSIndex : public VecSimTieredIndex { auto index = static_cast *>(idx); assert(index); // prevent parallel updates - std::lock_guard lock(index->updateJobMutex); - // Update the SVS index - index->updateSVSIndex(availableThreads); + std::lock_guard lock(index->updateJobMutex); // Release the scheduled flag to allow scheduling again index->indexUpdateScheduled.clear(); + // Update the SVS index + index->updateSVSIndex(availableThreads); } static void executeInsertJobWrapper(AsyncJob *job) { @@ -885,7 +885,6 @@ class TieredSVSIndex : public VecSimTieredIndex { int addVector(const void *blob, labelType label) override { int ret = 0; auto svs_index = GetSVSIndex(); - size_t update_threshold = 0; size_t frontend_index_size = 0; // In-Place mode - add vector syncronously to the backend index. @@ -917,7 +916,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto storage_blob = this->frontendIndex->preprocessForStorage(blob); // prevent update job from running in parallel and lock any access to the backend // index - std::lock_guard lock(this->updateJobMutex); + std::lock_guard lock(this->updateJobMutex); // Set available thread count to 1 for single vector write-in-place operation. // This maintains the contract that single vector operations use exactly one thread. // TODO: Replace this setNumThreads(1) call with an assertion once we establish @@ -1047,7 +1046,6 @@ class TieredSVSIndex : public VecSimTieredIndex { int deleteVector(labelType label) override { int ret = 0; - auto svs_index = GetSVSIndex(); // Backend index deletions to be synchronized with the frontend index, // elsewhere there is the risk of labels duplication in both indices which can lead to wrong // results of topK queries. In such case we should behave as if InPlace mode is always set. @@ -1112,7 +1110,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // the entire training duration (which can take 40-85s on slow machines). // If the mutex is held, training is actively running, so we report // indexUpdateScheduled = true (BACKGROUND_INDEXING = 1). - std::unique_lock lock(this->updateJobMutex, std::try_to_lock); + std::unique_lock lock(this->updateJobMutex, std::try_to_lock); if (lock.owns_lock()) { svsTieredInfo.indexUpdateScheduled = this->indexUpdateScheduled.test() == VecSimBool_TRUE; From 71c92bdc5d46ece42939ccd8f372036222efba1b Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 8 Jul 2026 00:30:16 -0700 Subject: [PATCH 09/64] fixes; tests pass --- src/VecSim/algorithms/svs/svs.h | 285 ++++++++++++------ .../algorithms/svs/svs_serializer_impl.h | 1 + src/VecSim/algorithms/svs/svs_tiered.h | 264 +++++++++------- tests/unit/test_svs_tiered.cpp | 15 +- 4 files changed, 360 insertions(+), 205 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index a3f2ca8c5..7dcdc1d93 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -39,6 +39,7 @@ struct SVSIndexBase virtual ~SVSIndexBase() = default; virtual int addVector(const void *vector_data, labelType label) = 0; virtual int addVectors(const void *vectors_data, const labelType *labels, size_t n) = 0; + virtual int deleteVector(labelType label) = 0; virtual int deleteVectors(const labelType *labels, size_t n) = 0; virtual bool isLabelExists(labelType label) const = 0; virtual size_t indexStorageSize() const = 0; @@ -46,6 +47,8 @@ struct SVSIndexBase virtual void setNumThreads(size_t numThreads) = 0; virtual size_t getThreadPoolCapacity() const = 0; virtual bool isCompressed() const = 0; + virtual bool ready() const = 0; + size_t getNumMarkedDeleted() const { return num_marked_deleted.load(std::memory_order_relaxed); } @@ -116,6 +119,17 @@ class SVSIndex : public VecSimIndexAbstract, fl svs::logging::logger_ptr logger_; // SVS Index implementation instance std::unique_ptr impl_; + mutable std::shared_mutex pimplGuard_; + + std::atomic impl_ready_{false}; + + void setReady() { + this->impl_ready_.store(true, std::memory_order_release); + } + + void setUnready() { + this->impl_ready_.store(false, std::memory_order_release); + } static double toVecSimDistance(float v) { return svs_details::toVecSimDistance(v); } @@ -243,7 +257,7 @@ class SVSIndex : public VecSimIndexAbstract, fl } void setImpl(std::unique_ptr handler) override { - if (impl_ != nullptr) { + if (ready()) { throw std::logic_error("SVSIndex::setImpl called on non-empty impl_"); } @@ -251,7 +265,11 @@ class SVSIndex : public VecSimIndexAbstract, fl if (!svs_handler) { throw std::logic_error("Failed to cast to SVSImplHandler"); } - this->impl_ = std::move(svs_handler->impl); + { + std::lock_guard lock(this->pimplGuard_); + this->impl_ = std::move(svs_handler->impl); + if (this->impl_) setReady(); + } } // Assuming numThreads was updated to reflect the number of available threads before this @@ -273,16 +291,23 @@ class SVSIndex : public VecSimIndexAbstract, fl // Wrap data into SVS SimpleDataView for SVS API auto points = svs::data::SimpleDataView{typed_vectors_data, n, this->dim}; - if (!impl_) { - // SVS index instance cannot be empty, so we have to construct it at first rows - impl_ = initImpl(points, ids); - } else { - if constexpr (!isMulti) { + if constexpr (!isMulti) { + if (ready()) { // SVS index does not support overriding vectors with the same label // so we have to delete them first if needed deleted_num = deleteVectorsImpl(labels, n); } + } + + if (!ready()) { + // SVS index instance cannot be empty, so we have to construct it at first rows + std::lock_guard lock(this->pimplGuard_); + impl_ = initImpl(points, ids); + assert(this->impl_ != nullptr); + setReady(); + } else { // Add new points to existing SVS index + std::shared_lock lock(this->pimplGuard_); impl_->add_points(points, ids); } @@ -294,7 +319,11 @@ class SVSIndex : public VecSimIndexAbstract, fl return 0; } - const auto deleted_num = impl_->delete_entries(std::span{&label, 1}); + int deleted_num = 0; + { + std::shared_lock lock(this->pimplGuard_); + deleted_num = impl_->delete_entries(std::span{&label, 1}); + } this->markIndexUpdate(deleted_num); return deleted_num; @@ -305,7 +334,11 @@ class SVSIndex : public VecSimIndexAbstract, fl return 0; } - const auto deleted_num = impl_->delete_entries(std::span{labels, n}); + int deleted_num = 0; + { + std::shared_lock lock(this->pimplGuard_); + deleted_num = impl_->delete_entries(std::span{labels, n}); + } this->markIndexUpdate(deleted_num); return deleted_num; @@ -313,12 +346,16 @@ class SVSIndex : public VecSimIndexAbstract, fl // Count deletions and consolidate index if needed void markIndexUpdate(size_t n = 1) { - if (!impl_) + if (!ready()) return; // SVS index instance should not be empty if (indexLabelCount() == 0) { - this->impl_.reset(); + std::lock_guard lock(this->pimplGuard_); + { + setUnready(); + this->impl_.reset(); + } num_marked_deleted.store(0, std::memory_order_relaxed); return; } @@ -351,32 +388,59 @@ class SVSIndex : public VecSimIndexAbstract, fl svs_details::getOrDefault(params.leanvec_dim, SVS_VAMANA_DEFAULT_LEANVEC_DIM)}, epsilon{svs_details::getOrDefault(params.epsilon, SVS_VAMANA_DEFAULT_EPSILON)}, is_two_level_lvq{isTwoLevelLVQ(params.quantBits)}, - threadpool_{1}, + threadpool_{std::max(size_t{SVS_VAMANA_DEFAULT_NUM_THREADS}, params.num_threads)}, impl_{nullptr} { logger_ = makeLogger(); } ~SVSIndex() = default; + bool ready() const override { + return this->impl_ready_.load(std::memory_order_acquire); + } + size_t indexSize() const override { return indexStorageSize(); } - size_t indexStorageSize() const override { return impl_ ? impl_->view_data().size() : 0; } + size_t indexStorageSize() const override { + if (ready()) { + std::shared_lock lock(this->pimplGuard_); + return impl_->view_data().size(); + } else { + return 0; + } + } size_t indexCapacity() const override { - return impl_ ? storage_traits_t::storage_capacity(impl_->view_data()) : 0; + if (ready()) { + std::shared_lock lock(this->pimplGuard_); + return storage_traits_t::storage_capacity(impl_->view_data()); + } else { + return 0; + } } size_t indexLabelCount() const override { if constexpr (isMulti) { - return impl_ ? impl_->labelcount() : 0; + if (ready()) { + std::shared_lock lock(this->pimplGuard_); + return impl_->labelcount(); + } else { + return 0; + } } else { - return impl_ ? impl_->size() : 0; + if (ready()) { + std::shared_lock lock(this->pimplGuard_); + return impl_->size(); + } else { + return 0; + } } } vecsim_stl::set getLabelsSet() const override { vecsim_stl::set labels(this->allocator); - if (impl_) { + if (ready()) { + std::shared_lock lock(this->pimplGuard_); impl_->on_ids([&labels](size_t label) { labels.insert(label); }); } return labels; @@ -528,7 +592,12 @@ class SVSIndex : public VecSimIndexAbstract, fl } bool isLabelExists(labelType label) const override { - return impl_ ? impl_->has_id(label) : false; + if (ready()) { + std::shared_lock lock(this->pimplGuard_); + return impl_->has_id(label); + } else { + return false; + } } size_t getNumThreads() const override { return threadpool_.size(); } @@ -543,13 +612,23 @@ class SVSIndex : public VecSimIndexAbstract, fl } double getDistanceFrom_Unsafe(labelType label, const void *vector_data) const override { - if (!impl_ || !impl_->has_id(label)) { + if (!ready()) { return std::numeric_limits::quiet_NaN(); - }; + } + { + std::shared_lock lock(this->pimplGuard_); + if (!impl_->has_id(label)) { + return std::numeric_limits::quiet_NaN(); + } + } + auto query_datum = std::span{static_cast(vector_data), this->dim}; - auto dist = impl_->get_distance(label, query_datum); - return toVecSimDistance(dist); + { + std::shared_lock lock(this->pimplGuard_); + auto dist = impl_->get_distance(label, query_datum); + return toVecSimDistance(dist); + } } VecSimQueryReply *topKQuery(const void *queryBlob, size_t k, @@ -559,36 +638,38 @@ class SVSIndex : public VecSimIndexAbstract, fl if (k == 0 || this->indexLabelCount() == 0) { return rep; } + { + std::shared_lock lock(this->pimplGuard_); + // limit result size to index size + k = std::min(k, this->indexLabelCount()); + + auto processed_query_ptr = this->preprocessQuery(queryBlob); + const void *processed_query = processed_query_ptr.get(); + + auto query = svs::data::ConstSimpleDataView{ + static_cast(processed_query), 1, this->dim}; + auto result = svs::QueryResult{query.size(), k}; + auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams, + is_two_level_lvq); + + auto timeoutCtx = queryParams ? queryParams->timeoutCtx : nullptr; + auto cancel = [timeoutCtx]() { return VECSIM_TIMEOUT(timeoutCtx); }; + + impl_->search(result.view(), query, sp, cancel); + if (cancel()) { + rep->code = VecSim_QueryReply_TimedOut; + return rep; + } - // limit result size to index size - k = std::min(k, this->indexLabelCount()); - - auto processed_query_ptr = this->preprocessQuery(queryBlob); - const void *processed_query = processed_query_ptr.get(); - - auto query = svs::data::ConstSimpleDataView{ - static_cast(processed_query), 1, this->dim}; - auto result = svs::QueryResult{query.size(), k}; - auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams, - is_two_level_lvq); - - auto timeoutCtx = queryParams ? queryParams->timeoutCtx : nullptr; - auto cancel = [timeoutCtx]() { return VECSIM_TIMEOUT(timeoutCtx); }; - - impl_->search(result.view(), query, sp, cancel); - if (cancel()) { - rep->code = VecSim_QueryReply_TimedOut; - return rep; - } - - assert(result.n_queries() == 1); + assert(result.n_queries() == 1); - const auto n_neighbors = result.n_neighbors(); - rep->results.reserve(n_neighbors); + const auto n_neighbors = result.n_neighbors(); + rep->results.reserve(n_neighbors); - for (size_t i = 0; i < n_neighbors; i++) { - rep->results.push_back( - VecSimQueryResult{result.index(0, i), toVecSimDistance(result.distance(0, i))}); + for (size_t i = 0; i < n_neighbors; i++) { + rep->results.push_back( + VecSimQueryResult{result.index(0, i), toVecSimDistance(result.distance(0, i))}); + } } // Workaround for VecSim merge_results() that expects results to be sorted // by score, then by id from both indices. @@ -604,58 +685,61 @@ class SVSIndex : public VecSimIndexAbstract, fl if (radius == 0 || this->indexLabelCount() == 0) { return rep; } + { + std::shared_lock lock(this->pimplGuard_); + + auto timeoutCtx = queryParams ? queryParams->timeoutCtx : nullptr; + auto cancel = [timeoutCtx]() { return VECSIM_TIMEOUT(timeoutCtx); }; + + // Prepare query blob for SVS + auto processed_query_ptr = this->preprocessQuery(queryBlob); + const void *processed_query = processed_query_ptr.get(); + std::span query{static_cast(processed_query), + this->dim}; + + // Base search parameters for the SVS iterator schedule. + auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams, + is_two_level_lvq); + // SVS BatchIterator handles the search in batches + // The batch size is set to the index search window size by default + const size_t batch_size = sp.buffer_config_.get_search_window_size(); + + // Create SVS BatchIterator for range search + // Search result is cached in the iterator and can be accessed by the user + auto svs_it = impl_->make_batch_iterator(query); + svs_it.next(batch_size, cancel); + if (cancel()) { + rep->code = VecSim_QueryReply_TimedOut; + return rep; + } - auto timeoutCtx = queryParams ? queryParams->timeoutCtx : nullptr; - auto cancel = [timeoutCtx]() { return VECSIM_TIMEOUT(timeoutCtx); }; - - // Prepare query blob for SVS - auto processed_query_ptr = this->preprocessQuery(queryBlob); - const void *processed_query = processed_query_ptr.get(); - std::span query{static_cast(processed_query), - this->dim}; - - // Base search parameters for the SVS iterator schedule. - auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams, - is_two_level_lvq); - // SVS BatchIterator handles the search in batches - // The batch size is set to the index search window size by default - const size_t batch_size = sp.buffer_config_.get_search_window_size(); - - // Create SVS BatchIterator for range search - // Search result is cached in the iterator and can be accessed by the user - auto svs_it = impl_->make_batch_iterator(query); - svs_it.next(batch_size, cancel); - if (cancel()) { - rep->code = VecSim_QueryReply_TimedOut; - return rep; - } - - // range search using epsilon - const auto epsilon = queryParams && queryParams->svsRuntimeParams.epsilon != 0 - ? queryParams->svsRuntimeParams.epsilon - : this->epsilon; - - const auto range_search_boundaries = radius * (1.0 + std::abs(epsilon)); - bool keep_searching = true; - - // Loop while iterator cache is not empty and search radius + epsilon is not exceeded - while (keep_searching && svs_it.size() > 0) { - // Iterate over the cached search results - for (auto &neighbor : svs_it) { - const auto dist = toVecSimDistance(neighbor.distance()); - if (dist <= radius) { - rep->results.push_back(VecSimQueryResult{neighbor.id(), dist}); - } else if (dist > range_search_boundaries) { - keep_searching = false; + // range search using epsilon + const auto epsilon = queryParams && queryParams->svsRuntimeParams.epsilon != 0 + ? queryParams->svsRuntimeParams.epsilon + : this->epsilon; + + const auto range_search_boundaries = radius * (1.0 + std::abs(epsilon)); + bool keep_searching = true; + + // Loop while iterator cache is not empty and search radius + epsilon is not exceeded + while (keep_searching && svs_it.size() > 0) { + // Iterate over the cached search results + for (auto &neighbor : svs_it) { + const auto dist = toVecSimDistance(neighbor.distance()); + if (dist <= radius) { + rep->results.push_back(VecSimQueryResult{neighbor.id(), dist}); + } else if (dist > range_search_boundaries) { + keep_searching = false; + } } - } - // If search radius + epsilon is not exceeded, request SVS BatchIterator for the next - // batch - if (keep_searching) { - svs_it.next(batch_size, cancel); - if (cancel()) { - rep->code = VecSim_QueryReply_TimedOut; - return rep; + // If search radius + epsilon is not exceeded, request SVS BatchIterator for the next + // batch + if (keep_searching) { + svs_it.next(batch_size, cancel); + if (cancel()) { + rep->code = VecSim_QueryReply_TimedOut; + return rep; + } } } } @@ -713,7 +797,8 @@ class SVSIndex : public VecSimIndexAbstract, fl } void runGC() override { - if (impl_) { + if (ready()) { + std::shared_lock lock(this->pimplGuard_); // There is documentation for consolidate(): // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.consolidate impl_->consolidate(); diff --git a/src/VecSim/algorithms/svs/svs_serializer_impl.h b/src/VecSim/algorithms/svs/svs_serializer_impl.h index 2780d3457..6f6757922 100644 --- a/src/VecSim/algorithms/svs/svs_serializer_impl.h +++ b/src/VecSim/algorithms/svs/svs_serializer_impl.h @@ -112,6 +112,7 @@ void SVSIndex distance_f(), std::move(threadpool_handle), false, logger_); impl_ = std::make_unique(std::move(loaded)); } + setReady(); } template { std::atomic_flag indexGCScheduled = ATOMIC_FLAG_INIT; // Used to prevent running multiple index update jobs in parallel. // Even if update jobs scheduled sequentially, they can be started in parallel. - mutable std::mutex updateJobMutex; + mutable std::shared_mutex updateJobMutex; // The reason of following container just to properly destroy jobs which not executed yet SVSMultiThreadJob::JobsRegistry uncompletedJobs; - std::atomic backendReady{false}; std::atomic backendInitSubmited{false}; + std::unordered_set ids_to_init_; vecsim_stl::unordered_map> labelToInsertJobs; // A mapping to hold invalid jobs, so we can dispose them upon index deletion. @@ -570,7 +570,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto index = static_cast *>(idx); assert(index); // prevent parallel updates - std::lock_guard lock(index->updateJobMutex); + std::lock_guard lock(index->updateJobMutex); // Release the scheduled flag to allow scheduling again index->indexUpdateScheduled.clear(); // Update the SVS index @@ -580,6 +580,8 @@ class TieredSVSIndex : public VecSimTieredIndex { static void executeInsertJobWrapper(AsyncJob *job) { auto *insert_job = static_cast(job); auto *job_index = static_cast *>(insert_job->index); + // prevent parallel execution with index initilizing job + std::shared_lock lock(job_index->updateJobMutex); job_index->executeInsertJob(insert_job); delete job; } @@ -705,7 +707,8 @@ class TieredSVSIndex : public VecSimTieredIndex { } void executeInsertJob(SVSInsertJob *job) { - assert(this->backendReady.load(std::memory_order_acquire)); + auto svs_index = GetSVSIndex(); + assert(svs_index->ready()); // Note that accessing the job fields should occur with flat index guard held (here and later). this->flatIndexGuard.lock_shared(); @@ -721,7 +724,6 @@ class TieredSVSIndex : public VecSimTieredIndex { auto storage_blob = this->frontendIndex->preprocessForStorage(this->frontendIndex->getDataByInternalId(job->id)); this->flatIndexGuard.unlock_shared(); - auto svs_index = GetSVSIndex(); svs_index->addVector(storage_blob.get(), job->label); // Remove the vector and the insert job from the flat buffer. @@ -763,84 +765,79 @@ class TieredSVSIndex : public VecSimTieredIndex { } void updateSVSIndex(size_t availableThreads) { + std::vector ids_to_move; std::vector labels_to_move; std::vector vectors_to_move; - std::vector deleted_labels_during_update; { // lock frontendIndex from modifications - std::shared_lock flat_lock{this->flatIndexGuard}; + std::lock_guard flat_lock{this->flatIndexGuard}; + auto svs_index = GetSVSIndex(); + bool is_initialization = !svs_index->ready(); auto flat_index = this->GetFlatIndex(); const auto frontend_index_size = this->frontendIndex->indexSize(); + const auto init_batch_size = is_initialization ? ids_to_init_.size() : frontend_index_size; const size_t dim = flat_index->getDim(); - labels_to_move.reserve(frontend_index_size); - vectors_to_move.reserve(frontend_index_size * dim); + ids_to_move.reserve(init_batch_size); + labels_to_move.reserve(init_batch_size); + vectors_to_move.reserve(init_batch_size * dim); + + if (is_initialization) { + for (idType id : ids_to_init_) { + ids_to_move.push_back(id); + labels_to_move.push_back(flat_index->getVectorLabel(id)); + auto data = flat_index->getDataByInternalId(id); + vectors_to_move.insert(vectors_to_move.end(), data, data + dim); + } + ids_to_init_.clear(); + } else { + for (idType id = 0; id < frontend_index_size; id++) { + ids_to_move.push_back(id); + labels_to_move.push_back(flat_index->getVectorLabel(id)); + auto data = flat_index->getDataByInternalId(id); + vectors_to_move.insert(vectors_to_move.end(), data, data + dim); + } + } - for (idType i = 0; i < frontend_index_size; i++) { - labels_to_move.push_back(flat_index->getVectorLabel(i)); - auto data = flat_index->getDataByInternalId(i); - vectors_to_move.insert(vectors_to_move.end(), data, data + dim); + std::sort(ids_to_move.begin(), ids_to_move.end()); + int total_deleted = 0; + for (auto it = ids_to_move.rbegin(); it != ids_to_move.rend(); ++it) { + idType id = *it; + auto label = this->frontendIndex->getVectorLabel(id); + // Delete the vector from the frontend index if not in-place updated. + if (label != SKIP_LABEL) { + labelType last_vec_label = + this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); + int deleted = this->frontendIndex->deleteVectorById(label, id); + if (deleted && id != this->frontendIndex->indexSize()) { + // If the vector removal caused a swap with the last id, update the relevant insert job. + this->updateInsertJobInternalId(this->frontendIndex->indexSize(), id, + last_vec_label); + } + total_deleted += deleted; + } } - // reset journal to the current frontend index state - swaps_journal.clear(); - deleted_labels_journal.clear(); + + assert(total_deleted == std::count_if(labels_to_move.begin(), labels_to_move.end(), + [](labelType label) { return label != SKIP_LABEL; }) && + "Deleted vectors count does not match the number of labels to delete"); + } // release frontend index executeTracingCallback("UpdateJob::before_add_to_svs"); { // lock backend index for writing and add vectors there auto svs_index = GetSVSIndex(); + svs_index->setNumThreads(1); assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); if (this->backendIndex->indexSize() == 0) { auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); - svs_index->setImpl(std::move(impl)); } else { svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); } } - - executeTracingCallback("UpdateJob::after_add_to_svs"); - // clean-up frontend index - { // lock frontend index for writing and delete moved vectors - std::lock_guard lock(this->flatIndexGuard); - - // swap deleted labels journal with the local variable to track deleted labels during - // update - std::swap(deleted_labels_during_update, deleted_labels_journal); - - // Apply swaps from journal to labels_to_move to reflect changes made in meanwhile. - applySwapsToLabelsArray(labels_to_move, this->swaps_journal); - - // delete vectors from the frontend index in reverse order - // it increases the chance of avoiding swaps in the frontend index and performance - // improvement - int deleted = 0; - idType id = labels_to_move.size(); - while (id-- > 0) { - auto label = labels_to_move[id]; - // Delete the vector from the frontend index if not in-place updated. - if (label != SKIP_LABEL) { - deleted += this->frontendIndex->deleteVectorById(label, id); - } - } - assert(deleted == std::count_if(labels_to_move.begin(), labels_to_move.end(), - [](labelType label) { return label != SKIP_LABEL; }) && - "Deleted vectors count does not match the number of labels to delete"); - } - // delete vectors from backend index that were deleted from the frontend index during - // the update process. - { - std::sort(deleted_labels_during_update.begin(), deleted_labels_during_update.end()); - auto it = std::unique(deleted_labels_during_update.begin(), - deleted_labels_during_update.end()); - deleted_labels_during_update.erase(it, deleted_labels_during_update.end()); - auto svs_index = GetSVSIndex(); - svs_index->deleteVectors(deleted_labels_during_update.data(), - deleted_labels_during_update.size()); - } - this->backendReady.store(true, std::memory_order_release); } public: @@ -916,7 +913,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto storage_blob = this->frontendIndex->preprocessForStorage(blob); // prevent update job from running in parallel and lock any access to the backend // index - std::lock_guard lock(this->updateJobMutex); + std::lock_guard lock(this->updateJobMutex); // Set available thread count to 1 for single vector write-in-place operation. // This maintains the contract that single vector operations use exactly one thread. // TODO: Replace this setNumThreads(1) call with an assertion once we establish @@ -931,16 +928,11 @@ class TieredSVSIndex : public VecSimTieredIndex { if (!this->backendInitSubmited.load(std::memory_order_acquire)) { // Add vector to the frontend index. std::lock_guard lock(this->flatIndexGuard); - const auto ft_ret = this->frontendIndex->addVector(blob, label); - - if (ft_ret == 0) { // Vector was overriden - add 'skiping' swap to the journal. - assert(!this->backendIndex->isMultiValue() && - "addVector() may return 0 for single value indices only"); - for (auto id : this->frontendIndex->getElementIds(label)) { - this->swaps_journal.emplace_back(SKIP_LABEL, id, id); - } - deleted_labels_journal.push_back(label); + if (this->frontendIndex->isLabelExists(label)) { + deleteAndUpdateInitIds(label); } + ids_to_init_.insert(this->frontendIndex->indexSize()); + const auto ft_ret = this->frontendIndex->addVector(blob, label); ret = std::max(ret + ft_ret, 0); if (this->frontendIndex->indexSize() >= this->trainingTriggerThreshold) { @@ -950,13 +942,8 @@ class TieredSVSIndex : public VecSimTieredIndex { return ret; } - // spin-lock, while backend is initilizing - while (!this->backendReady.load(std::memory_order_acquire)) { - __builtin_ia32_pause(); - } - this->flatIndexGuard.lock_shared(); - if (this->frontendIndex->indexSize() >= flat_buffer_bound) { + if (svs_index->ready() && (this->frontendIndex->indexSize() >= flat_buffer_bound)) { this->flatIndexGuard.unlock_shared(); auto storage_blob = this->frontendIndex->preprocessForStorage(blob); @@ -966,7 +953,18 @@ class TieredSVSIndex : public VecSimTieredIndex { this->flatIndexGuard.lock(); idType new_flat_id = this->frontendIndex->indexSize(); if (this->frontendIndex->isLabelExists(label) && !this->frontendIndex->isMultiValue()) { + if (this->labelToInsertJobs.count(label) == 0) { + // No pending insert job. + // Just replace vector in the frontend + // If this label already exists, this will do overwrite. + ret = this->frontendIndex->addVector(blob, label); + this->flatIndexGuard.unlock(); + return ret; + } + // Overwrite the vector and invalidate its only pending job (since we are not in MULTI). + // Label exists, but job doesn't. It means the label is used for initialization. + // Just create new job, this job will remove duplicate from the backend. auto *old_job = this->labelToInsertJobs.at(label).at(0); old_job->id = this->setAndSaveInvalidJob(old_job); this->labelToInsertJobs.erase(label); @@ -998,19 +996,26 @@ class TieredSVSIndex : public VecSimTieredIndex { } this->flatIndexGuard.unlock(); + // Here, a worker might ingest the previous vector that was stored under "label" + // (in case of override in non-MULTI index) - so if it's there, we remove it + // we submit the insert job. + if (!this->backendIndex->isMultiValue()) { + if (svs_index->ready()) { + // If we removed the previous vector from both svs and flat in the overwrite process, + // we still return 0 (not -1). + ret = std::max(ret - svs_index->deleteVector(label), 0); + } + } + // Insert job to the queue and signal the workers' updater. this->submitSingleJob(new_insert_job); return ret; } } - int deleteAndRecordSwaps_Unsafe(labelType label) { + void deleteAndUpdateInitIds(labelType label) { auto deleting_ids = this->frontendIndex->getElementIds(label); - // assert if all elements of deleting_ids are unique - assert(std::set(deleting_ids.begin(), deleting_ids.end()).size() == deleting_ids.size() && - "deleting_ids should contain unique ids"); - // Sort deleting_ids by id descending order std::sort(deleting_ids.begin(), deleting_ids.end(), [](const auto &a, const auto &b) { return a > b; }); @@ -1025,43 +1030,94 @@ class TieredSVSIndex : public VecSimTieredIndex { }) && "updated_ids should be a subset of deleting_ids"); - // Record swaps in the journal. - for (auto id : deleting_ids) { - auto it = updated_ids.find(id); - if (it != updated_ids.end()) { - assert(id == it->first && "id in updated_ids should match the id in deleting_ids"); - auto newId = id; - auto oldId = it->second.first; - auto oldLabel = it->second.second; - this->swaps_journal.emplace_back(oldLabel, oldId, newId); - } else { - // No swap, just delete is marked by oldId == newId == deleted id - this->swaps_journal.emplace_back(SKIP_LABEL, id, id); + std::vector new_ids_to_init; + for (auto &it : updated_ids) { + idType prev_id = it.second.first; + idType new_id = it.first; + + if (ids_to_init_.count(prev_id) > 0) { + ids_to_init_.erase(prev_id); + new_ids_to_init.push_back(new_id); + } + labelType updated_vec_label = it.second.second; + this->updateInsertJobInternalId(prev_id, new_id, updated_vec_label); + } + + for (idType new_id : new_ids_to_init) { + ids_to_init_.insert(new_id); + } + } + + int deleteAndRecordSwaps_Unsafe(labelType label) { + auto deleting_ids = this->frontendIndex->getElementIds(label); + if (deleting_ids.size() == 0) return 0; + + // assert if all elements of deleting_ids are unique + assert(std::set(deleting_ids.begin(), deleting_ids.end()).size() == deleting_ids.size() && + "deleting_ids should contain unique ids"); + + // If id is deleted, don't use it for initialization + for (idType id : deleting_ids) { + ids_to_init_.erase(id); + } + + if (this->labelToInsertJobs.count(label) > 0) { + // Invalidate the pending insert job(s) into SVS associated with this label + auto &insert_jobs = this->labelToInsertJobs.at(label); + for (auto *job : insert_jobs) { + job->id = this->setAndSaveInvalidJob(job); } + // Remove the pending insert job(s) from the labelToInsertJobs mapping. + this->labelToInsertJobs.erase(label); } - deleted_labels_journal.push_back(label); + deleteAndUpdateInitIds(label); return deleting_ids.size(); } int deleteVector(labelType label) override { int ret = 0; - // Backend index deletions to be synchronized with the frontend index, - // elsewhere there is the risk of labels duplication in both indices which can lead to wrong - // results of topK queries. In such case we should behave as if InPlace mode is always set. - bool label_exists = [&]() { - std::shared_lock lock(this->flatIndexGuard); - return this->frontendIndex->isLabelExists(label); - }(); - - if (label_exists) { - std::lock_guard lock(this->flatIndexGuard); - ret = this->deleteAndRecordSwaps_Unsafe(label); + + this->flatIndexGuard.lock_shared(); + if (this->frontendIndex->isLabelExists(label)) { + this->flatIndexGuard.unlock_shared(); + this->flatIndexGuard.lock(); + // Check again if the label exists, as it may have been removed while we released the lock. + if (this->frontendIndex->isLabelExists(label)) { + ret = this->deleteAndRecordSwaps_Unsafe(label); + // if (this->labelToInsertJobs.count(label) > 0) { + // // Invalidate the pending insert job(s) into SVS associated with this label + // auto &insert_jobs = this->labelToInsertJobs.at(label); + // for (auto *job : insert_jobs) { + // job->id = this->setAndSaveInvalidJob(job); + // } + // ret += insert_jobs.size(); + // // Remove the pending insert job(s) from the labelToInsertJobs mapping. + // this->labelToInsertJobs.erase(label); + // // Go over the every id that corresponds the label and remove it from the flat buffer. + // // Every delete may cause a swap of the deleted id with the last id, and we return a + // // mapping from id to the original id that resides in this id after the deletion(s) (see + // // an example in this function implementation in MULTI index). + // auto updated_ids = this->frontendIndex->deleteVectorAndGetUpdatedIds(label); + // for (auto &it : updated_ids) { + // idType prev_id = it.second.first; + // labelType updated_vec_label = it.second.second; + // this->updateInsertJobInternalId(prev_id, it.first, updated_vec_label); + // } + // } else { + // // label was deleted during initilization stage + // ret = this->deleteAndRecordSwaps_Unsafe(label); + // } + } + this->flatIndexGuard.unlock(); + } else { + this->flatIndexGuard.unlock_shared(); } ret += this->backendIndex->deleteVector(label); return ret; } + size_t getNumMarkedDeleted() const override { return this->GetSVSIndex()->getNumMarkedDeleted(); } @@ -1110,7 +1166,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // the entire training duration (which can take 40-85s on slow machines). // If the mutex is held, training is actively running, so we report // indexUpdateScheduled = true (BACKGROUND_INDEXING = 1). - std::unique_lock lock(this->updateJobMutex, std::try_to_lock); + std::unique_lock lock(this->updateJobMutex, std::try_to_lock); if (lock.owns_lock()) { svsTieredInfo.indexUpdateScheduled = this->indexUpdateScheduled.test() == VecSimBool_TRUE; diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 3d9c0bd0d..63f9c6a23 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -613,6 +613,12 @@ TYPED_TEST(SVSTieredIndexTest, background_indexing_check) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); } + // The backgroundIndexing flag tracks the batch update job only; individual async + // SVS_INSERT_VECTOR_JOBs may still be in flight, each transiently holding its vector in + // both the flat and backend indices. Drain the queue so the total size is stable before + // asserting the exact count. + mock_thread_pool.thread_pool_join(); + ASSERT_GT(tiered_index->GetBackendIndex()->indexSize(), training_th + second_batch / update_th); ASSERT_LT(tiered_index->GetFlatIndex()->indexSize(), update_th); ASSERT_EQ(tiered_index->indexSize(), second_batch + training_th); @@ -1125,7 +1131,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, markedDeleted) { ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 0); // Move vectors to the backend - mock_thread_pool.thread_iteration(); + while (mock_thread_pool.jobQ.size() > 0) mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), n); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->indexSize(), n); @@ -1222,13 +1228,20 @@ TYPED_TEST(SVSTieredIndexTestBasic, deleteVectorMulti) { ASSERT_EQ(tiered_index->indexSize(), 2); ASSERT_EQ(tiered_index->deleteVector(vec_label), 2); ASSERT_EQ(tiered_index->indexLabelCount(), 0); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 2); + mock_thread_pool.thread_iteration(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); mock_thread_pool.thread_iteration(); ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); // Test deleting a label for which both of its vector's is in SVS index. GenerateAndAddVector(tiered_index, dim, vec_label, vec_label); GenerateAndAddVector(tiered_index, dim, vec_label, other_vec_val); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 2); mock_thread_pool.thread_iteration(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); + mock_thread_pool.thread_iteration(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); ASSERT_EQ(tiered_index->indexLabelCount(), 1); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 2); From b130e31c684c9647107ddc75fdd24028a34ee019 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 8 Jul 2026 06:47:45 -0700 Subject: [PATCH 10/64] refactoring --- src/VecSim/algorithms/svs/svs.h | 4 +- src/VecSim/algorithms/svs/svs_tiered.h | 162 ++++++++++--------------- tests/unit/test_svs_fp16.cpp | 2 +- tests/unit/test_svs_tiered.cpp | 4 +- 4 files changed, 67 insertions(+), 105 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 7dcdc1d93..ee71b7470 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -340,7 +340,9 @@ class SVSIndex : public VecSimIndexAbstract, fl deleted_num = impl_->delete_entries(std::span{labels, n}); } - this->markIndexUpdate(deleted_num); + if (deleted_num > 0) + this->markIndexUpdate(deleted_num); + return deleted_num; } diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 8869b2191..8608de84c 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -232,7 +232,7 @@ class TieredSVSIndex : public VecSimTieredIndex { using swap_record = std::tuple; constexpr static size_t SKIP_LABEL = std::numeric_limits::max(); std::vector swaps_journal; - // deleted_labels_journal is used by updateSVSIndex() to track vectors that were deleted from + // deleted_labels_journal is used by initSVSIndex() to track vectors that were deleted from // Flat index during SVS index updating. The journal contains the deleted labels. These labels // are used to delete the same vectors from the SVS index at the end of the update. std::vector deleted_labels_journal; @@ -553,19 +553,19 @@ class TieredSVSIndex : public VecSimTieredIndex { private: /** - * @brief Updates the SVS index in a thread-safe manner. + * @brief Init the SVS index in a thread-safe manner. * * This static wrapper function performs the following actions: * - Acquires a lock on the index's updateJobMutex to prevent concurrent updates. * - Clears the indexUpdateScheduled flag to allow future scheduling. * - Configures the number of threads for the underlying SVS index update operation. - * - Calls the updateSVSIndex method to perform the actual index update. + * - Calls the initSVSIndex method to perform the actual index update. * * @param idx Pointer to the VecSimIndex to be updated. * @param availableThreads The number of threads available for the update operation. Current * thread us used as well, so the minimal value is 1. */ - static void updateSVSIndexWrapper(VecSimIndex *idx, size_t availableThreads) { + static void initSVSIndexWrapper(VecSimIndex *idx, size_t availableThreads) { assert(availableThreads > 0); auto index = static_cast *>(idx); assert(index); @@ -574,7 +574,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // Release the scheduled flag to allow scheduling again index->indexUpdateScheduled.clear(); // Update the SVS index - index->updateSVSIndex(availableThreads); + index->initSVSIndex(availableThreads); } static void executeInsertJobWrapper(AsyncJob *job) { @@ -640,7 +640,7 @@ class TieredSVSIndex : public VecSimTieredIndex { } } - void scheduleSVSIndexUpdate() { + void scheduleSVSIndexInit() { // do not schedule if scheduled already if (indexUpdateScheduled.test_and_set()) { return; @@ -648,7 +648,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto total_threads = this->GetSVSIndex()->getThreadPoolCapacity(); auto jobs = SVSMultiThreadJob::createJobs( - this->allocator, SVS_BATCH_UPDATE_JOB, updateSVSIndexWrapper, this, total_threads, + this->allocator, SVS_BATCH_UPDATE_JOB, initSVSIndexWrapper, this, total_threads, std::chrono::microseconds(updateJobWaitTime), &uncompletedJobs); this->submitJobs(jobs); } @@ -764,39 +764,38 @@ class TieredSVSIndex : public VecSimTieredIndex { this->flatIndexGuard.unlock(); } - void updateSVSIndex(size_t availableThreads) { + void initSVSIndex(size_t availableThreads) { std::vector ids_to_move; std::vector labels_to_move; std::vector vectors_to_move; { // lock frontendIndex from modifications + // The whole initialization is done under flatIndexGuard std::lock_guard flat_lock{this->flatIndexGuard}; - auto svs_index = GetSVSIndex(); - bool is_initialization = !svs_index->ready(); auto flat_index = this->GetFlatIndex(); - const auto frontend_index_size = this->frontendIndex->indexSize(); - const auto init_batch_size = is_initialization ? ids_to_init_.size() : frontend_index_size; + const auto init_batch_size = ids_to_init_.size(); const size_t dim = flat_index->getDim(); ids_to_move.reserve(init_batch_size); labels_to_move.reserve(init_batch_size); vectors_to_move.reserve(init_batch_size * dim); - if (is_initialization) { - for (idType id : ids_to_init_) { - ids_to_move.push_back(id); - labels_to_move.push_back(flat_index->getVectorLabel(id)); - auto data = flat_index->getDataByInternalId(id); - vectors_to_move.insert(vectors_to_move.end(), data, data + dim); - } - ids_to_init_.clear(); - } else { - for (idType id = 0; id < frontend_index_size; id++) { - ids_to_move.push_back(id); - labels_to_move.push_back(flat_index->getVectorLabel(id)); - auto data = flat_index->getDataByInternalId(id); - vectors_to_move.insert(vectors_to_move.end(), data, data + dim); - } + for (idType id : ids_to_init_) { + ids_to_move.push_back(id); + labels_to_move.push_back(flat_index->getVectorLabel(id)); + auto data = flat_index->getDataByInternalId(id); + vectors_to_move.insert(vectors_to_move.end(), data, data + dim); + } + ids_to_init_.clear(); + + executeTracingCallback("UpdateJob::before_add_to_svs"); + { + auto svs_index = GetSVSIndex(); + svs_index->setNumThreads(1); + assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); + auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), + labels_to_move.size()); + svs_index->setImpl(std::move(impl)); } std::sort(ids_to_move.begin(), ids_to_move.end()); @@ -821,23 +820,7 @@ class TieredSVSIndex : public VecSimTieredIndex { assert(total_deleted == std::count_if(labels_to_move.begin(), labels_to_move.end(), [](labelType label) { return label != SKIP_LABEL; }) && "Deleted vectors count does not match the number of labels to delete"); - } // release frontend index - - executeTracingCallback("UpdateJob::before_add_to_svs"); - { // lock backend index for writing and add vectors there - auto svs_index = GetSVSIndex(); - svs_index->setNumThreads(1); - assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); - if (this->backendIndex->indexSize() == 0) { - auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), - labels_to_move.size()); - svs_index->setImpl(std::move(impl)); - } else { - svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), - labels_to_move.size()); - } - } } public: @@ -887,25 +870,29 @@ class TieredSVSIndex : public VecSimTieredIndex { // In-Place mode - add vector syncronously to the backend index. if (this->getWriteMode() == VecSim_WriteInPlace) { // It is ok to lock everything at once for in-place mode, - // but we will have to unlock averything before calling updateSVSIndexWrapper() + // but we will have to unlock averything before calling initSVSIndexWrapper() // so make the minimal needed lock here. // Backend index initialization data have to be buffered for proper // compression/training. if (this->backendIndex->indexSize() == 0) { // If backend index size is 0, first collect vectors in frontend index // lock in scope to ensure that these will be released before - // updateSVSIndexWrapper() is called. + // initSVSIndexWrapper() is called. { std::lock_guard lock(this->flatIndexGuard); + if (this->frontendIndex->isLabelExists(label)) { + deleteAndUpdateInitIds(label); + } + ids_to_init_.insert(this->frontendIndex->indexSize()); ret = this->frontendIndex->addVector(blob, label); // If frontend size exceeds the update job threshold, ... frontend_index_size = this->frontendIndex->indexSize(); } // ... move vectors to the backend index. if (frontend_index_size >= this->trainingTriggerThreshold) { - // updateSVSIndexWrapper() accures it's own locks + // initSVSIndexWrapper() accures it's own locks // initialize the SVS index synchonously using current thread only - updateSVSIndexWrapper(this, 1); + initSVSIndexWrapper(this, 1); } return ret; } else { @@ -937,7 +924,7 @@ class TieredSVSIndex : public VecSimTieredIndex { if (this->frontendIndex->indexSize() >= this->trainingTriggerThreshold) { this->backendInitSubmited.store(true, std::memory_order_release); - scheduleSVSIndexUpdate(); + scheduleSVSIndexInit(); } return ret; } @@ -1048,33 +1035,6 @@ class TieredSVSIndex : public VecSimTieredIndex { } } - int deleteAndRecordSwaps_Unsafe(labelType label) { - auto deleting_ids = this->frontendIndex->getElementIds(label); - if (deleting_ids.size() == 0) return 0; - - // assert if all elements of deleting_ids are unique - assert(std::set(deleting_ids.begin(), deleting_ids.end()).size() == deleting_ids.size() && - "deleting_ids should contain unique ids"); - - // If id is deleted, don't use it for initialization - for (idType id : deleting_ids) { - ids_to_init_.erase(id); - } - - if (this->labelToInsertJobs.count(label) > 0) { - // Invalidate the pending insert job(s) into SVS associated with this label - auto &insert_jobs = this->labelToInsertJobs.at(label); - for (auto *job : insert_jobs) { - job->id = this->setAndSaveInvalidJob(job); - } - // Remove the pending insert job(s) from the labelToInsertJobs mapping. - this->labelToInsertJobs.erase(label); - } - - deleteAndUpdateInitIds(label); - return deleting_ids.size(); - } - int deleteVector(labelType label) override { int ret = 0; @@ -1084,30 +1044,30 @@ class TieredSVSIndex : public VecSimTieredIndex { this->flatIndexGuard.lock(); // Check again if the label exists, as it may have been removed while we released the lock. if (this->frontendIndex->isLabelExists(label)) { - ret = this->deleteAndRecordSwaps_Unsafe(label); - // if (this->labelToInsertJobs.count(label) > 0) { - // // Invalidate the pending insert job(s) into SVS associated with this label - // auto &insert_jobs = this->labelToInsertJobs.at(label); - // for (auto *job : insert_jobs) { - // job->id = this->setAndSaveInvalidJob(job); - // } - // ret += insert_jobs.size(); - // // Remove the pending insert job(s) from the labelToInsertJobs mapping. - // this->labelToInsertJobs.erase(label); - // // Go over the every id that corresponds the label and remove it from the flat buffer. - // // Every delete may cause a swap of the deleted id with the last id, and we return a - // // mapping from id to the original id that resides in this id after the deletion(s) (see - // // an example in this function implementation in MULTI index). - // auto updated_ids = this->frontendIndex->deleteVectorAndGetUpdatedIds(label); - // for (auto &it : updated_ids) { - // idType prev_id = it.second.first; - // labelType updated_vec_label = it.second.second; - // this->updateInsertJobInternalId(prev_id, it.first, updated_vec_label); - // } - // } else { - // // label was deleted during initilization stage - // ret = this->deleteAndRecordSwaps_Unsafe(label); - // } + auto deleting_ids = this->frontendIndex->getElementIds(label); + if (deleting_ids.size() == 0) return 0; + + // assert if all elements of deleting_ids are unique + assert(std::set(deleting_ids.begin(), deleting_ids.end()).size() == deleting_ids.size() && + "deleting_ids should contain unique ids"); + + // If id is deleted, don't use it for initialization + for (idType id : deleting_ids) { + ids_to_init_.erase(id); + } + + if (this->labelToInsertJobs.count(label) > 0) { + // Invalidate the pending insert job(s) into SVS associated with this label + auto &insert_jobs = this->labelToInsertJobs.at(label); + for (auto *job : insert_jobs) { + job->id = this->setAndSaveInvalidJob(job); + } + // Remove the pending insert job(s) from the labelToInsertJobs mapping. + this->labelToInsertJobs.erase(label); + } + + deleteAndUpdateInitIds(label); + return deleting_ids.size(); } this->flatIndexGuard.unlock(); } else { @@ -1162,7 +1122,7 @@ class TieredSVSIndex : public VecSimTieredIndex { }; { // Use try_lock to avoid blocking the main thread during long-running - // training operations. updateSVSIndexWrapper holds updateJobMutex for + // training operations. initSVSIndexWrapper holds updateJobMutex for // the entire training duration (which can take 40-85s on slow machines). // If the mutex is held, training is actively running, so we report // indexUpdateScheduled = true (BACKGROUND_INDEXING = 1). @@ -1171,7 +1131,7 @@ class TieredSVSIndex : public VecSimTieredIndex { svsTieredInfo.indexUpdateScheduled = this->indexUpdateScheduled.test() == VecSimBool_TRUE; } else { - // Mutex is held by updateSVSIndexWrapper — training is in progress. + // Mutex is held by initSVSIndexWrapper — training is in progress. svsTieredInfo.indexUpdateScheduled = true; } } diff --git a/tests/unit/test_svs_fp16.cpp b/tests/unit/test_svs_fp16.cpp index 59c11c0a1..ded23bd72 100644 --- a/tests/unit/test_svs_fp16.cpp +++ b/tests/unit/test_svs_fp16.cpp @@ -2337,7 +2337,7 @@ class FP16SVSTieredIndexTest : public FP16SVSTest { } } // Submit the index update job. - tiered_index->scheduleSVSIndexUpdate(); + tiered_index->scheduleSVSIndexInit(); ASSERT_EQ(mock_thread_pool.jobQ.size(), mock_thread_pool.thread_pool_size); // Execute the job from the queue and validate that the index was updated properly. diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 63f9c6a23..2980d0e38 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -489,7 +489,7 @@ TYPED_TEST(SVSTieredIndexTest, CreateIndexInstance) { ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 0); // Submit the index update job. - tiered_index->scheduleSVSIndexUpdate(); + tiered_index->scheduleSVSIndexInit(); ASSERT_EQ(mock_thread_pool.jobQ.size(), mock_thread_pool.thread_pool_size); // Execute the job from the queue and validate that the index was updated properly. @@ -1635,7 +1635,7 @@ TYPED_TEST(SVSTieredIndexTest, parallelInsertAdHoc) { tiered_index->submitSingleJob(search_job); } - tiered_index->scheduleSVSIndexUpdate(); + tiered_index->scheduleSVSIndexInit(); mock_thread_pool.thread_pool_join(); EXPECT_EQ(successful_searches, n); From 78b4ef0286899cecd13f248985152031fcdf0de9 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 15 Jul 2026 09:27:43 -0700 Subject: [PATCH 11/64] fixes for GC --- src/VecSim/algorithms/svs/svs_tiered.h | 46 ++++++++++++++++++-------- src/VecSim/vec_sim_common.h | 1 + 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 8608de84c..2e797e188 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -25,6 +25,15 @@ struct SVSInsertJob : public AsyncJob { : AsyncJob(allocator, SVS_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} }; +/** + * Definition of a job that launches GC on SVS Index. + */ +struct SVSGCJob : public AsyncJob { + SVSGCJob(std::shared_ptr allocator, + JobCallback insertCb, VecSimIndex *index_) + : AsyncJob(allocator, SVS_GC2_JOB, insertCb, index_) {} +}; + /** * @class SVSMultiThreadJob @@ -601,13 +610,17 @@ class TieredSVSIndex : public VecSimTieredIndex { * @note no need to implement extra non-static method, as GC logic is simple enough to be done * here. */ - static void SVSIndexGCWrapper(VecSimIndex *idx, size_t availableThreads) { - assert(availableThreads > 0); - auto index = static_cast *>(idx); - assert(index); + // static void SVSIndexGCWrapper(VecSimIndex *idx) { + static void SVSIndexGCWrapper(AsyncJob *job) { + // assert(availableThreads > 0); + // auto index = static_cast *>(idx); + // assert(index); - // Release the scheduled flag to allow scheduling again - index->indexGCScheduled.clear(); + auto gc_job = static_cast(job); + auto index = static_cast *>(gc_job->index); + + // // Release the scheduled flag to allow scheduling again + // index->indexGCScheduled.clear(); // Do SVS index GC index->backendIndex->log(VecSimCommonStrings::LOG_VERBOSE_STRING, @@ -617,9 +630,13 @@ class TieredSVSIndex : public VecSimTieredIndex { // No need to run GC on an empty index. return; } - svs_index->setNumThreads(std::min(availableThreads, index->backendIndex->indexSize())); + // svs_index->setNumThreads(std::min(availableThreads, index->backendIndex->indexSize())); + svs_index->setNumThreads(1); // VecSimIndexAbstract::runGC() is protected static_cast(index->backendIndex)->runGC(); + + // Release the scheduled flag to allow scheduling again + index->indexGCScheduled.clear(); } #ifdef BUILD_TESTS @@ -659,11 +676,11 @@ class TieredSVSIndex : public VecSimTieredIndex { return; } - auto total_threads = this->GetSVSIndex()->getThreadPoolCapacity(); - auto jobs = SVSMultiThreadJob::createJobs( - this->allocator, SVS_GC_JOB, SVSIndexGCWrapper, this, total_threads, - std::chrono::microseconds(updateJobWaitTime), &uncompletedJobs); - this->submitJobs(jobs); + AsyncJob *new_GC_job = new (this->allocator) + SVSGCJob(this->allocator, SVSIndexGCWrapper, this); + + // Insert job to the queue. + this->submitSingleJob(new_GC_job); } private: @@ -724,6 +741,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto storage_blob = this->frontendIndex->preprocessForStorage(this->frontendIndex->getDataByInternalId(job->id)); this->flatIndexGuard.unlock_shared(); + svs_index->setNumThreads(1); svs_index->addVector(storage_blob.get(), job->label); // Remove the vector and the insert job from the flat buffer. @@ -933,6 +951,7 @@ class TieredSVSIndex : public VecSimTieredIndex { if (svs_index->ready() && (this->frontendIndex->indexSize() >= flat_buffer_bound)) { this->flatIndexGuard.unlock_shared(); auto storage_blob = this->frontendIndex->preprocessForStorage(blob); + svs_index->setNumThreads(1); return ret = svs_index->addVector(storage_blob.get(), label); } else { @@ -1041,7 +1060,7 @@ class TieredSVSIndex : public VecSimTieredIndex { this->flatIndexGuard.lock_shared(); if (this->frontendIndex->isLabelExists(label)) { this->flatIndexGuard.unlock_shared(); - this->flatIndexGuard.lock(); + std::lock_guard flat_lock{this->flatIndexGuard}; // Check again if the label exists, as it may have been removed while we released the lock. if (this->frontendIndex->isLabelExists(label)) { auto deleting_ids = this->frontendIndex->getElementIds(label); @@ -1069,7 +1088,6 @@ class TieredSVSIndex : public VecSimTieredIndex { deleteAndUpdateInitIds(label); return deleting_ids.size(); } - this->flatIndexGuard.unlock(); } else { this->flatIndexGuard.unlock_shared(); } diff --git a/src/VecSim/vec_sim_common.h b/src/VecSim/vec_sim_common.h index 03e7870f3..d800e7cdc 100644 --- a/src/VecSim/vec_sim_common.h +++ b/src/VecSim/vec_sim_common.h @@ -275,6 +275,7 @@ typedef enum { SVS_BATCH_UPDATE_JOB, SVS_INSERT_VECTOR_JOB, SVS_GC_JOB, + SVS_GC2_JOB, INVALID_JOB // to indicate that finding a JobType >= INVALID_JOB is an error } JobType; From fedd22ea70579aa50f9d75b88cce536066f5ea55 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 20 Jul 2026 09:35:50 -0700 Subject: [PATCH 12/64] copy blob before flat unlock --- src/VecSim/algorithms/svs/svs_tiered.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 2e797e188..4ebca3e09 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -738,11 +738,16 @@ class TieredSVSIndex : public VecSimTieredIndex { this->invalidJobsLookupGuard.unlock(); return; } - auto storage_blob = this->frontendIndex->preprocessForStorage(this->frontendIndex->getDataByInternalId(job->id)); + + // Copy the vector blob out of the flat buffer while holding flatIndexGuard, so we + // can release the flat lock before indexing into the SVS backend + size_t data_size = this->frontendIndex->getStoredDataSize(); + auto blob_copy = this->getAllocator()->allocate_unique(data_size); + memcpy(blob_copy.get(), this->frontendIndex->getDataByInternalId(job->id), data_size); this->flatIndexGuard.unlock_shared(); svs_index->setNumThreads(1); - svs_index->addVector(storage_blob.get(), job->label); + svs_index->addVector(blob_copy.get(), job->label); // Remove the vector and the insert job from the flat buffer. this->flatIndexGuard.lock(); @@ -812,7 +817,7 @@ class TieredSVSIndex : public VecSimTieredIndex { svs_index->setNumThreads(1); assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), - labels_to_move.size()); + labels_to_move.size()); svs_index->setImpl(std::move(impl)); } @@ -1315,3 +1320,4 @@ class TieredSVSIndex : public VecSimTieredIndex { this->flatIndexGuard.unlock_shared(); } }; + \ No newline at end of file From fcf83dd767e08a8b7747ad16d46736576a9a6f58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mihai=20Capot=C4=83?= Date: Thu, 23 Jul 2026 22:09:56 -0700 Subject: [PATCH 13/64] Remove null characters from end of file --- src/VecSim/algorithms/svs/svs_tiered.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 4ebca3e09..e3eb441b8 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -1320,4 +1320,3 @@ class TieredSVSIndex : public VecSimTieredIndex { this->flatIndexGuard.unlock_shared(); } }; - \ No newline at end of file From 4035699ee202da005643963d268d94573e8e9f19 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Tue, 28 Jul 2026 02:28:19 -0700 Subject: [PATCH 14/64] fixes for gc --- src/VecSim/algorithms/svs/svs.h | 60 ++++++++++--- src/VecSim/algorithms/svs/svs_tiered.h | 109 +++++++++++++++++------ src/VecSim/vec_sim_common.h | 3 +- tests/unit/test_svs_fp16.cpp | 8 ++ tests/unit/test_svs_tiered.cpp | 116 ++++++++++++++----------- 5 files changed, 204 insertions(+), 92 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index ee71b7470..e410529f7 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -41,6 +41,7 @@ struct SVSIndexBase virtual int addVectors(const void *vectors_data, const labelType *labels, size_t n) = 0; virtual int deleteVector(labelType label) = 0; virtual int deleteVectors(const labelType *labels, size_t n) = 0; + virtual void consolidate(const std::vector& labels) = 0; virtual bool isLabelExists(labelType label) const = 0; virtual size_t indexStorageSize() const = 0; virtual size_t getNumThreads() const = 0; @@ -299,21 +300,36 @@ class SVSIndex : public VecSimIndexAbstract, fl } } + this->pimplGuard_.lock_shared(); if (!ready()) { + this->pimplGuard_.unlock_shared(); // SVS index instance cannot be empty, so we have to construct it at first rows std::lock_guard lock(this->pimplGuard_); - impl_ = initImpl(points, ids); - assert(this->impl_ != nullptr); - setReady(); + if (!ready()) { + impl_ = initImpl(points, ids); + assert(this->impl_ != nullptr); + setReady(); + } else { + impl_->add_points(points, ids); + } } else { // Add new points to existing SVS index - std::shared_lock lock(this->pimplGuard_); impl_->add_points(points, ids); + this->pimplGuard_.unlock_shared(); } return n - deleted_num; } + void consolidate(const std::vector& labels) { + std::shared_lock lock(this->pimplGuard_); + if (!ready()) + return; + + size_t n_consolidated = impl_->consolidate(labels); + num_marked_deleted.fetch_sub(n_consolidated, std::memory_order_relaxed); + } + int deleteVectorImpl(const labelType label) { if (indexLabelCount() == 0) { return 0; @@ -354,12 +370,14 @@ class SVSIndex : public VecSimIndexAbstract, fl // SVS index instance should not be empty if (indexLabelCount() == 0) { std::lock_guard lock(this->pimplGuard_); - { - setUnready(); - this->impl_.reset(); + if (indexLabelCountUnsafe() == 0) { + { + setUnready(); + this->impl_.reset(); + } + num_marked_deleted.store(0, std::memory_order_relaxed); + return; } - num_marked_deleted.store(0, std::memory_order_relaxed); - return; } num_marked_deleted.fetch_add(n, std::memory_order_relaxed); @@ -404,8 +422,8 @@ class SVSIndex : public VecSimIndexAbstract, fl size_t indexSize() const override { return indexStorageSize(); } size_t indexStorageSize() const override { + std::shared_lock lock(this->pimplGuard_); if (ready()) { - std::shared_lock lock(this->pimplGuard_); return impl_->view_data().size(); } else { return 0; @@ -413,14 +431,30 @@ class SVSIndex : public VecSimIndexAbstract, fl } size_t indexCapacity() const override { + std::shared_lock lock(this->pimplGuard_); if (ready()) { - std::shared_lock lock(this->pimplGuard_); return storage_traits_t::storage_capacity(impl_->view_data()); } else { return 0; } } + size_t indexLabelCountUnsafe() const { + if constexpr (isMulti) { + if (ready()) { + return impl_->labelcount(); + } else { + return 0; + } + } else { + if (ready()) { + return impl_->size(); + } else { + return 0; + } + } + } + size_t indexLabelCount() const override { if constexpr (isMulti) { if (ready()) { @@ -641,7 +675,6 @@ class SVSIndex : public VecSimIndexAbstract, fl return rep; } { - std::shared_lock lock(this->pimplGuard_); // limit result size to index size k = std::min(k, this->indexLabelCount()); @@ -651,6 +684,7 @@ class SVSIndex : public VecSimIndexAbstract, fl auto query = svs::data::ConstSimpleDataView{ static_cast(processed_query), 1, this->dim}; auto result = svs::QueryResult{query.size(), k}; + std::shared_lock lock(this->pimplGuard_); auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams, is_two_level_lvq); @@ -803,7 +837,7 @@ class SVSIndex : public VecSimIndexAbstract, fl std::shared_lock lock(this->pimplGuard_); // There is documentation for consolidate(): // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.consolidate - impl_->consolidate(); + // impl_->consolidate(); // There is documentation for compact(): // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.compact impl_->compact(); diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 4ebca3e09..c94d3b711 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -25,13 +25,24 @@ struct SVSInsertJob : public AsyncJob { : AsyncJob(allocator, SVS_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} }; +/** + * Definition of a job that launches partial consolidation on SVS Index. + */ +struct SVSConsolidateJob : public AsyncJob { + std::vector labels; + + SVSConsolidateJob(std::shared_ptr allocator, const std::vector& labels_, + JobCallback insertCb, VecSimIndex *index_) + : AsyncJob(allocator, SVS_CONSOLIDATE_JOB, insertCb, index_), labels(labels_) {} +}; + /** * Definition of a job that launches GC on SVS Index. */ struct SVSGCJob : public AsyncJob { SVSGCJob(std::shared_ptr allocator, JobCallback insertCb, VecSimIndex *index_) - : AsyncJob(allocator, SVS_GC2_JOB, insertCb, index_) {} + : AsyncJob(allocator, SVS_GC_JOB, insertCb, index_) {} }; @@ -248,6 +259,7 @@ class TieredSVSIndex : public VecSimTieredIndex { size_t trainingTriggerThreshold; size_t updateTriggerThreshold; + size_t consolidateTriggerThreshold; size_t updateJobWaitTime; // Used to prevent scheduling multiple index update jobs at the same time. // As far as the update job does a batch update, job queue should have just 1 job at the moment. @@ -263,6 +275,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::atomic backendInitSubmited{false}; std::unordered_set ids_to_init_; + std::vector labels_to_consolidate_; vecsim_stl::unordered_map> labelToInsertJobs; // A mapping to hold invalid jobs, so we can dispose them upon index deletion. @@ -610,23 +623,18 @@ class TieredSVSIndex : public VecSimTieredIndex { * @note no need to implement extra non-static method, as GC logic is simple enough to be done * here. */ - // static void SVSIndexGCWrapper(VecSimIndex *idx) { static void SVSIndexGCWrapper(AsyncJob *job) { - // assert(availableThreads > 0); - // auto index = static_cast *>(idx); - // assert(index); - auto gc_job = static_cast(job); auto index = static_cast *>(gc_job->index); - // // Release the scheduled flag to allow scheduling again - // index->indexGCScheduled.clear(); - + std::shared_lock lock(index->updateJobMutex); // Do SVS index GC index->backendIndex->log(VecSimCommonStrings::LOG_VERBOSE_STRING, "running asynchronous GC for tiered SVS index"); auto svs_index = index->GetSVSIndex(); if (index->backendIndex->indexSize() == 0) { + index->indexGCScheduled.clear(); + delete job; // No need to run GC on an empty index. return; } @@ -637,6 +645,18 @@ class TieredSVSIndex : public VecSimTieredIndex { // Release the scheduled flag to allow scheduling again index->indexGCScheduled.clear(); + delete job; + } + + static void SVSIndexConsolidateWrapper(AsyncJob *job) { + auto consolidate_job = static_cast(job); + auto index = static_cast *>(consolidate_job->index); + + std::shared_lock lock(index->updateJobMutex); + auto svs_index = index->GetSVSIndex(); + svs_index->setNumThreads(1); + svs_index->consolidate(consolidate_job->labels); + delete job; } #ifdef BUILD_TESTS @@ -683,6 +703,19 @@ class TieredSVSIndex : public VecSimTieredIndex { this->submitSingleJob(new_GC_job); } + void scheduleSVSIndexConsolidate(labelType label) { + labels_to_consolidate_.push_back(label); + + if (labels_to_consolidate_.size() >= consolidateTriggerThreshold) { + AsyncJob *new_consolidate_job = new (this->allocator) + SVSConsolidateJob(this->allocator, labels_to_consolidate_, SVSIndexConsolidateWrapper, this); + + // Insert job to the queue. + this->submitSingleJob(new_consolidate_job); + labels_to_consolidate_.clear(); + } + } + private: static void applySwapsToLabelsArray(std::vector &labels, const std::vector &swaps) { @@ -746,7 +779,7 @@ class TieredSVSIndex : public VecSimTieredIndex { memcpy(blob_copy.get(), this->frontendIndex->getDataByInternalId(job->id), data_size); this->flatIndexGuard.unlock_shared(); - svs_index->setNumThreads(1); + // svs_index->setNumThreads(1); svs_index->addVector(blob_copy.get(), job->label); // Remove the vector and the insert job from the flat buffer. @@ -792,6 +825,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::vector labels_to_move; std::vector vectors_to_move; + executeTracingCallback("UpdateJob::before_add_to_svs"); { // lock frontendIndex from modifications // The whole initialization is done under flatIndexGuard std::lock_guard flat_lock{this->flatIndexGuard}; @@ -811,13 +845,13 @@ class TieredSVSIndex : public VecSimTieredIndex { } ids_to_init_.clear(); - executeTracingCallback("UpdateJob::before_add_to_svs"); { auto svs_index = GetSVSIndex(); - svs_index->setNumThreads(1); + svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); + svs_index->setNumThreads(1); svs_index->setImpl(std::move(impl)); } @@ -844,6 +878,7 @@ class TieredSVSIndex : public VecSimTieredIndex { [](labelType label) { return label != SKIP_LABEL; }) && "Deleted vectors count does not match the number of labels to delete"); } // release frontend index + executeTracingCallback("UpdateJob::after_add_to_svs"); } public: @@ -872,6 +907,8 @@ class TieredSVSIndex : public VecSimTieredIndex { ? SVS_VAMANA_DEFAULT_TRAINING_THRESHOLD : this->updateTriggerThreshold; + this->consolidateTriggerThreshold = SVS_VAMANA_DEFAULT_CONSOLIDATE_THRESHOLD; + this->trainingTriggerThreshold = tiered_svs_params.trainingTriggerThreshold == 0 ? default_training_threshold @@ -897,7 +934,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // so make the minimal needed lock here. // Backend index initialization data have to be buffered for proper // compression/training. - if (this->backendIndex->indexSize() == 0) { + if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { // If backend index size is 0, first collect vectors in frontend index // lock in scope to ensure that these will be released before // initSVSIndexWrapper() is called. @@ -929,16 +966,22 @@ class TieredSVSIndex : public VecSimTieredIndex { // TODO: Replace this setNumThreads(1) call with an assertion once we establish // a contract that write-in-place mode guarantees numThreads == 1. svs_index->setNumThreads(1); - return this->backendIndex->addVector(storage_blob.get(), label); + int deleted = 0; + if (!this->backendIndex->isMultiValue()) { + deleted = svs_index->deleteVector(label); + if (deleted > 0) + svs_index->consolidate({label}); + } + return this->backendIndex->addVector(storage_blob.get(), label) - deleted; } } assert(this->getWriteMode() != VecSim_WriteInPlace && "InPlace mode returns early"); // Async mode - add vector to the frontend index and schedule an update job if needed. - if (!this->backendInitSubmited.load(std::memory_order_acquire)) { + if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { // Add vector to the frontend index. std::lock_guard lock(this->flatIndexGuard); - if (this->frontendIndex->isLabelExists(label)) { + if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { deleteAndUpdateInitIds(label); } ids_to_init_.insert(this->frontendIndex->indexSize()); @@ -956,7 +999,12 @@ class TieredSVSIndex : public VecSimTieredIndex { if (svs_index->ready() && (this->frontendIndex->indexSize() >= flat_buffer_bound)) { this->flatIndexGuard.unlock_shared(); auto storage_blob = this->frontendIndex->preprocessForStorage(blob); - svs_index->setNumThreads(1); + + if (!this->backendIndex->isMultiValue()) { + auto deleted = svs_index->deleteVector(label); + if (deleted > 0) + scheduleSVSIndexConsolidate(label); + } return ret = svs_index->addVector(storage_blob.get(), label); } else { @@ -989,7 +1037,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // increase index capacity. } // If this label already exists, this will do overwrite. - this->frontendIndex->addVector(blob, label); + ret += this->frontendIndex->addVector(blob, label); AsyncJob *new_insert_job = new (this->allocator) SVSInsertJob(this->allocator, label, new_flat_id, executeInsertJobWrapper, this); @@ -1014,7 +1062,10 @@ class TieredSVSIndex : public VecSimTieredIndex { if (svs_index->ready()) { // If we removed the previous vector from both svs and flat in the overwrite process, // we still return 0 (not -1). - ret = std::max(ret - svs_index->deleteVector(label), 0); + auto deleted = svs_index->deleteVector(label); + if (deleted > 0) + scheduleSVSIndexConsolidate(label); + ret = std::max(ret - deleted, 0); } } @@ -1091,13 +1142,21 @@ class TieredSVSIndex : public VecSimTieredIndex { } deleteAndUpdateInitIds(label); - return deleting_ids.size(); + ret += deleting_ids.size(); } } else { this->flatIndexGuard.unlock_shared(); } - ret += this->backendIndex->deleteVector(label); + auto deleted = this->backendIndex->deleteVector(label); + if (deleted > 0) { + if (this->getWriteMode() == VecSim_WriteInPlace) { + GetSVSIndex()->consolidate({label}); + } else { + scheduleSVSIndexConsolidate(label); + } + } + ret += deleted; return ret; } @@ -1159,10 +1218,11 @@ class TieredSVSIndex : public VecSimTieredIndex { } } info.tieredInfo.specificTieredBackendInfo.svsTieredInfo = svsTieredInfo; + // Background indexing is in progress whenever the flat buffer is non-empty: every vector + // buffered there has either a pending per-vector insert job or a batch-init job that will + // drain it into the backend. info.tieredInfo.backgroundIndexing = - svsTieredInfo.indexUpdateScheduled && info.tieredInfo.frontendCommonInfo.indexSize > 0 - ? VecSimBool_TRUE - : VecSimBool_FALSE; + info.tieredInfo.frontendCommonInfo.indexSize > 0 ? VecSimBool_TRUE : VecSimBool_FALSE; return info; } @@ -1320,4 +1380,3 @@ class TieredSVSIndex : public VecSimTieredIndex { this->flatIndexGuard.unlock_shared(); } }; - \ No newline at end of file diff --git a/src/VecSim/vec_sim_common.h b/src/VecSim/vec_sim_common.h index d800e7cdc..7d87baea0 100644 --- a/src/VecSim/vec_sim_common.h +++ b/src/VecSim/vec_sim_common.h @@ -50,6 +50,7 @@ extern "C" { #define SVS_VAMANA_DEFAULT_TRAINING_THRESHOLD (10 * DEFAULT_BLOCK_SIZE) // 10 * 1024 vectors // Default batch update threshold for SVS index. #define SVS_VAMANA_DEFAULT_UPDATE_THRESHOLD (1 * DEFAULT_BLOCK_SIZE) // 1 * 1024 vectors +#define SVS_VAMANA_DEFAULT_CONSOLIDATE_THRESHOLD (1 * DEFAULT_BLOCK_SIZE) // 1 * 1024 vectors #define SVS_VAMANA_DEFAULT_SEARCH_WINDOW_SIZE 10 // NOTE: No need to have SVS_VAMANA_DEFAULT_SEARCH_BUFFER_CAPACITY // as the default is determined by the search_window_size @@ -275,7 +276,7 @@ typedef enum { SVS_BATCH_UPDATE_JOB, SVS_INSERT_VECTOR_JOB, SVS_GC_JOB, - SVS_GC2_JOB, + SVS_CONSOLIDATE_JOB, INVALID_JOB // to indicate that finding a JobType >= INVALID_JOB is an error } JobType; diff --git a/tests/unit/test_svs_fp16.cpp b/tests/unit/test_svs_fp16.cpp index ded23bd72..95a26e6c2 100644 --- a/tests/unit/test_svs_fp16.cpp +++ b/tests/unit/test_svs_fp16.cpp @@ -2919,13 +2919,21 @@ TYPED_TEST(FP16SVSTieredIndexTest, deleteVectorMulti) { ASSERT_EQ(tiered_index->deleteVector(vec_label), 2); ASSERT_EQ(tiered_index->indexLabelCount(), 0); + + ASSERT_EQ(mock_thread_pool.jobQ.size(), 2); + mock_thread_pool.thread_iteration(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); mock_thread_pool.thread_iteration(); ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); // Test deleting a label for which both of its vector's is in SVS index. this->GenerateAndAddVector(tiered_index, dim, vec_label, vec_label); this->GenerateAndAddVector(tiered_index, dim, vec_label, other_vec_val); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 2); mock_thread_pool.thread_iteration(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); + mock_thread_pool.thread_iteration(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); ASSERT_EQ(tiered_index->indexLabelCount(), 1); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 2); diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 2980d0e38..6ef887f79 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -556,14 +556,16 @@ TYPED_TEST(SVSTieredIndexTest, addVector) { ASSERT_LE(expected_mem, tiered_index->getAllocationSize()); if constexpr (TypeParam::isMulti()) { - // Add another vector under the same label + // Add another vector under the same label. The backend init was already submitted + // by the first vector (thread_pool_size batch-init jobs), so this second vector takes + // the per-vector async path and schedules one additional SVSInsertJob of its own. VecSimIndex_AddVector(tiered_index, vector, vec_label); ASSERT_EQ(tiered_index->indexSize(), 2); ASSERT_EQ(tiered_index->indexLabelCount(), 1); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 2); - // Validate that there still 1 update jobs set - ASSERT_EQ(mock_thread_pool.jobQ.size(), mock_thread_pool.thread_pool_size); + // The batch-init jobs plus the extra per-vector insert job. + ASSERT_EQ(mock_thread_pool.jobQ.size(), mock_thread_pool.thread_pool_size + 1); } } @@ -2937,12 +2939,13 @@ TYPED_TEST(SVSTieredIndexTest, writeInPlaceMode) { ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 2); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); - // Overwrite inplace - only in single-value mode + // In write-in-place mode every mutation runs synchronously, including consolidation + // of soft-deleted slots. size_t expected_marked_deleted = 0; + // Overwrite inplace - only in single-value mode if (!TypeParam::isMulti()) { TEST_DATA_T overwritten_vec[] = {1, 1, 1, 1}; tiered_index->addVector(overwritten_vec, vec_label); - expected_marked_deleted++; ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 3); ASSERT_EQ(tiered_index->indexSize(), 3); ASSERT_EQ(tiered_index->indexLabelCount(), 2); @@ -2950,9 +2953,9 @@ TYPED_TEST(SVSTieredIndexTest, writeInPlaceMode) { ASSERT_EQ(tiered_index->getDistanceFrom_Unsafe(vec_label, overwritten_vec), 0); ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), expected_marked_deleted); } - // Validate that the vector is marked as deleted. + // Validate that the vector is deleted and consolidated synchronously (in-place mode), + // so no marked-deleted entry remains. tiered_index->deleteVector(vec_label); - expected_marked_deleted++; ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), TypeParam::isMulti() ? 2 : 3); ASSERT_EQ(tiered_index->indexLabelCount(), 1); @@ -3142,12 +3145,13 @@ TYPED_TEST(SVSTieredIndexTestBasic, runGCAPI) { auto jobs_before_gc = mock_thread_pool.jobQ.size(); // Run the GC API call, expect that we will clean up the SVS index. VecSimTieredIndex_GC(tiered_index); - // Expected that GC jobs were added to the queue. - ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + mock_thread_pool.thread_pool_size); + // Expected that a single GC job was added to the queue. + ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + 1); // Run GC twice. VecSimTieredIndex_GC(tiered_index); - // Expected that no new GC jobs were added to the queue. - ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + mock_thread_pool.thread_pool_size); + // Expected that no new GC job was added to the queue (indexGCScheduled is still set until the + // pending job runs). + ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + 1); // Wait for any pending jobs to complete. As far as SVS GC is done via a job. mock_thread_pool.init_threads(); mock_thread_pool.thread_pool_join(); @@ -3415,7 +3419,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalSingle) { // update job paused, we have vectors 0-(n-1) in the index, let's do index modifications // Remove vector label=n-2, it is copied to backend index. - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 2), 2); + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 2), 1); // Update vector label=1. EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, 1, 10), 0); // Add a new vector @@ -3423,9 +3427,9 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalSingle) { // Add another one EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, n + 1, n + 1), 1); // Remove vector label=0, it is copied to backend index. - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, 0), 2); + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, 0), 1); // Remove the last vector copied to backend index - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 1), 2); + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 1), 1); // Update vector label=2. EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, 2, 20), 0); // Remove vector label=2. @@ -3447,16 +3451,15 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalSingle) { // 0:deleted, 1: 10, 2: deleted, 3:3, ..., n-2:deleted n-1: 10(n-1), n+1: n+1; // total: n-2 vectors and labels ASSERT_EQ(tiered_index->indexLabelCount(), n - 2); - EXPECT_EQ(tiered_index->GetBackendIndex()->indexLabelCount(), n - 5); + // Nothing remains in flat. + EXPECT_EQ(tiered_index->GetBackendIndex()->indexLabelCount(), n - 2); - // We added 3 vectors to the flat index and removed 5 vectors from the backend index. - // Backend index: 0:deleted, 1:deleted, 2:deleted, 3:3, ..., n-2:deleted, n-1:deleted; - // total: n-5 - EXPECT_EQ(tiered_index->GetBackendIndex()->indexSize(), n); - ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 5); - // Frontend index: 1:10, n-1:10(n-1), n+1:n+1 - ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 3); - ASSERT_EQ(tiered_index->indexSize(), n + tiered_index->GetFlatIndex()->indexSize()); + EXPECT_EQ(tiered_index->GetBackendIndex()->indexSize(), n - 2); + ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 0); + // Flat buffer fully drained by the insert jobs. + ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); + ASSERT_EQ(tiered_index->indexSize(), tiered_index->GetBackendIndex()->indexSize() + + tiered_index->GetFlatIndex()->indexSize()); double abs_err = 1e-2; // Allow a larger relative error for quantization. TEST_DATA_T expected_vector[dim]; @@ -3469,7 +3472,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalSingle) { GenerateVector(expected_vector, dim, 0); ASSERT_TRUE(std::isnan(tiered_index->getDistanceFrom_Unsafe(n - 2, expected_vector))); - // Vector label=1, with value 10 should be in the flat index. + // Vector label=1 was updated to value 10; its latest value is retrievable from the index. GenerateVector(expected_vector, dim, 10); ASSERT_NEAR(tiered_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); @@ -3548,22 +3551,24 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalMulti) { // update job paused, we have vectors 0-(n-1) in the index, let's do index modifications - // Remove vector label=n-2, it is copied to backend index. - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 2), 2); + // Remove vector label=n-2. At the pause it lives only in the backend + //, so a single vector is removed. + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 2), 1); // Add one more vector label=1. EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, 1, 10), 1); // Add a new vector EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, n, n), 1); // Add another one EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, n + 1, n + 1), 1); - // Remove vector label=0, it is copied to backend index. - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, 0), 2); - // Remove the last vector copied to backend index - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 1), 2); + // Remove vector label=0, only in the backend at the pause. + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, 0), 1); + // Remove the last vector, only in the backend at the pause. + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 1), 1); // Add one more vector label=2. EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, 2, 20), 1); - // Remove vector label=2: for multi: old is copied to backend , old + new are in flat - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, 2), 3); + // Remove vector label=2: for multi both the original (2) and the new (20) copies sit in + // the flat buffer at this point, so 2 vectors are removed. + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, 2), 2); // Remove vector label=n - in flat only EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n), 1); // Add vector (n-1) again @@ -3581,15 +3586,16 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalMulti) { // 0: deleted, 1: (1,10), 2: deleted, 3:3, ..., n-2: deleted n-1: 10(n-1), n+1: n+1; // total: n-2 labels, n-1 vectors ASSERT_EQ(tiered_index->indexLabelCount(), n - 2); - EXPECT_EQ(tiered_index->GetBackendIndex()->indexLabelCount(), n - 4); + // In the async-insert design the per-vector insert jobs drain every surviving vector from + // the flat buffer into the backend, so nothing remains in flat. + EXPECT_EQ(tiered_index->GetBackendIndex()->indexLabelCount(), n - 2); - // We added 3 vectors to the flat index and removed 4 vectors from the backend index. - // Backend index: 0:deleted, 1:1, 2:deleted, 3:3, ..., n-2:deleted, n-1:deleted; total: n-4 - EXPECT_EQ(tiered_index->GetBackendIndex()->indexSize(), n); - ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 4); - // Frontend index: 1:10, n-1:10(n-1), n+1:n+1 - ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 3); - ASSERT_EQ(tiered_index->indexSize(), n + tiered_index->GetFlatIndex()->indexSize()); + EXPECT_EQ(tiered_index->GetBackendIndex()->indexSize(), n - 1); + ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 0); + // Flat buffer fully drained by the insert jobs. + ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); + ASSERT_EQ(tiered_index->indexSize(), tiered_index->GetBackendIndex()->indexSize() + + tiered_index->GetFlatIndex()->indexSize()); double abs_err = 1e-2; // Allow a larger relative error for quantization. TEST_DATA_T expected_vector[dim]; @@ -3602,11 +3608,11 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalMulti) { GenerateVector(expected_vector, dim, 0); ASSERT_TRUE(std::isnan(tiered_index->getDistanceFrom_Unsafe(n - 2, expected_vector))); - // There are 2 vectors labeled "1" with values 1 in backend and 10 in flat. - // We expect the minimal distance for the query 10 to be taken from flat index. + // Label "1" is multi-valued and keeps both of its vectors (values 1 and 10). + // The minimal distance for the query 10 should match the second value. GenerateVector(expected_vector, dim, 10); ASSERT_NEAR(tiered_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); - // And the minimal distance for the query 1.0 to be taken from backend + // And the minimal distance for the query 1.0 should match the first value. GenerateVector(expected_vector, dim, 1); ASSERT_NEAR(tiered_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); @@ -3706,8 +3712,9 @@ TYPED_TEST(SVSTieredIndexTestBasic, testDeletedJournalSingle) { mock_thread_pool.thread_pool_join(); - // Verify that vectors labels: {0, 1, 2, n-1} are marked as deleted in the SVS index. - ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 4); + // The update job is paused before it transfers vectors to the backend, so at the pause the + // backend is still empty. + ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 0); // Verify that the deleted vectors are not accessible. double abs_err = 1e-2; // Allow a larger relative error for quantization. @@ -3724,9 +3731,10 @@ TYPED_TEST(SVSTieredIndexTestBasic, testDeletedJournalSingle) { ASSERT_TRUE(std::isnan(tiered_index->getDistanceFrom_Unsafe(label, expected_vector))); } - // label 1 - updated to 10 but deleted in the SVS index - ASSERT_TRUE(flat_index->isLabelExists(1)); - ASSERT_FALSE(svs_index->isLabelExists(1)); + // label 1 - updated to 10 during the pause. The insert job drains it from the flat buffer + // into the backend, so it now lives in the SVS index (not flat) with its updated value. + ASSERT_FALSE(flat_index->isLabelExists(1)); + ASSERT_TRUE(svs_index->isLabelExists(1)); GenerateVector(expected_vector, dim, 10); ASSERT_NEAR(tiered_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); } @@ -3808,8 +3816,9 @@ TYPED_TEST(SVSTieredIndexTestBasic, testDeletedJournalMulti) { mock_thread_pool.thread_pool_join(); - // Verify that vectors labels: {0, 2, n-1} are marked as deleted in the SVS index. - ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 3); + // The update job is paused before it transfers vectors to the backend, so at the pause the + // backend is still empty. + ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 0); // Verify that the deleted vectors are not accessible. double abs_err = 1e-2; // Allow a larger relative error for quantization. @@ -3826,13 +3835,14 @@ TYPED_TEST(SVSTieredIndexTestBasic, testDeletedJournalMulti) { ASSERT_TRUE(std::isnan(tiered_index->getDistanceFrom_Unsafe(label, expected_vector))); } - // label 1 - multi-value 1 (in SVS) and 10 (in flat) - ASSERT_TRUE(flat_index->isLabelExists(1)); + // label 1 - multi-value with both vectors (1 and 10). The insert jobs drain the flat buffer + // into the backend, so both values now live in the SVS index and none remain in flat. + ASSERT_FALSE(flat_index->isLabelExists(1)); ASSERT_TRUE(svs_index->isLabelExists(1)); GenerateVector(expected_vector, dim, 1); ASSERT_NEAR(backend_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); GenerateVector(expected_vector, dim, 10); - ASSERT_NEAR(flat_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); + ASSERT_NEAR(backend_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); } TEST(SVSTieredIndexTest, testThreadPool) { From 649460a7a28553129a7c76b5eddde1c0bf99c85c Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 30 Jul 2026 00:17:00 -0700 Subject: [PATCH 15/64] fix writer starvation for compaction --- src/VecSim/algorithms/svs/svs_tiered.h | 64 ++++++++++++++++++++------ 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index c94d3b711..c72503829 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -623,29 +623,55 @@ class TieredSVSIndex : public VecSimTieredIndex { * @note no need to implement extra non-static method, as GC logic is simple enough to be done * here. */ - static void SVSIndexGCWrapper(AsyncJob *job) { - auto gc_job = static_cast(job); - auto index = static_cast *>(gc_job->index); + // static void SVSIndexGCWrapper(AsyncJob *job) { + // auto gc_job = static_cast(job); + // auto index = static_cast *>(gc_job->index); + + // std::shared_lock lock(index->updateJobMutex); + // // Do SVS index GC + // index->backendIndex->log(VecSimCommonStrings::LOG_VERBOSE_STRING, + // "running asynchronous GC for tiered SVS index"); + // auto svs_index = index->GetSVSIndex(); + // if (index->backendIndex->indexSize() == 0) { + // index->indexGCScheduled.clear(); + // delete job; + // // No need to run GC on an empty index. + // return; + // } + // // svs_index->setNumThreads(std::min(availableThreads, index->backendIndex->indexSize())); + // svs_index->setNumThreads(1); + // // VecSimIndexAbstract::runGC() is protected + // fprintf(stderr, "runGC\n"); + // static_cast(index->backendIndex)->runGC(); + // fprintf(stderr, "GC done\n"); + + // // Release the scheduled flag to allow scheduling again + // index->indexGCScheduled.clear(); + // delete job; + // } + + static void SVSIndexGCWrapper(VecSimIndex *idx, size_t availableThreads) { + assert(availableThreads > 0); + auto index = static_cast *>(idx); + assert(index); - std::shared_lock lock(index->updateJobMutex); // Do SVS index GC index->backendIndex->log(VecSimCommonStrings::LOG_VERBOSE_STRING, "running asynchronous GC for tiered SVS index"); auto svs_index = index->GetSVSIndex(); if (index->backendIndex->indexSize() == 0) { - index->indexGCScheduled.clear(); - delete job; // No need to run GC on an empty index. return; } - // svs_index->setNumThreads(std::min(availableThreads, index->backendIndex->indexSize())); - svs_index->setNumThreads(1); + std::lock_guard lock(index->updateJobMutex); + + svs_index->setNumThreads(std::min(availableThreads, index->backendIndex->indexSize())); // VecSimIndexAbstract::runGC() is protected static_cast(index->backendIndex)->runGC(); + svs_index->setNumThreads(1); // Release the scheduled flag to allow scheduling again index->indexGCScheduled.clear(); - delete job; } static void SVSIndexConsolidateWrapper(AsyncJob *job) { @@ -696,11 +722,17 @@ class TieredSVSIndex : public VecSimTieredIndex { return; } - AsyncJob *new_GC_job = new (this->allocator) - SVSGCJob(this->allocator, SVSIndexGCWrapper, this); + auto total_threads = this->GetSVSIndex()->getThreadPoolCapacity(); + auto jobs = SVSMultiThreadJob::createJobs( + this->allocator, SVS_GC_JOB, SVSIndexGCWrapper, this, total_threads, + std::chrono::microseconds(updateJobWaitTime), &uncompletedJobs); + this->submitJobs(jobs); + + // AsyncJob *new_GC_job = new (this->allocator) + // SVSGCJob(this->allocator, SVSIndexGCWrapper, this); - // Insert job to the queue. - this->submitSingleJob(new_GC_job); + // // Insert job to the queue. + // this->submitSingleJob(new_GC_job); } void scheduleSVSIndexConsolidate(labelType label) { @@ -780,7 +812,10 @@ class TieredSVSIndex : public VecSimTieredIndex { this->flatIndexGuard.unlock_shared(); // svs_index->setNumThreads(1); - svs_index->addVector(blob_copy.get(), job->label); + { + std::shared_lock lock(updateJobMutex); + svs_index->addVector(blob_copy.get(), job->label); + } // Remove the vector and the insert job from the flat buffer. this->flatIndexGuard.lock(); @@ -1006,6 +1041,7 @@ class TieredSVSIndex : public VecSimTieredIndex { scheduleSVSIndexConsolidate(label); } + std::shared_lock lock(updateJobMutex); return ret = svs_index->addVector(storage_blob.get(), label); } else { this->flatIndexGuard.unlock_shared(); From 07f2c8fdf3b7ab80d80c8e9488c8a0b75192d37d Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Tue, 11 Aug 2026 03:52:21 -0700 Subject: [PATCH 16/64] adopt sigle label consolidation --- src/VecSim/algorithms/svs/svs.h | 4 +-- src/VecSim/algorithms/svs/svs_tiered.h | 40 +++++++++++--------------- tests/unit/test_svs_fp16.cpp | 9 ++++-- tests/unit/test_svs_tiered.cpp | 18 +++++++----- 4 files changed, 36 insertions(+), 35 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index e410529f7..33cfbc334 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -310,11 +310,11 @@ class SVSIndex : public VecSimIndexAbstract, fl assert(this->impl_ != nullptr); setReady(); } else { - impl_->add_points(points, ids); + impl_->add_points(points, ids, /*reuse_empty*/ false); } } else { // Add new points to existing SVS index - impl_->add_points(points, ids); + impl_->add_points(points, ids, /*reuse_empty*/ false); this->pimplGuard_.unlock_shared(); } diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index c72503829..27d2f4d12 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -259,7 +259,6 @@ class TieredSVSIndex : public VecSimTieredIndex { size_t trainingTriggerThreshold; size_t updateTriggerThreshold; - size_t consolidateTriggerThreshold; size_t updateJobWaitTime; // Used to prevent scheduling multiple index update jobs at the same time. // As far as the update job does a batch update, job queue should have just 1 job at the moment. @@ -275,7 +274,6 @@ class TieredSVSIndex : public VecSimTieredIndex { std::atomic backendInitSubmited{false}; std::unordered_set ids_to_init_; - std::vector labels_to_consolidate_; vecsim_stl::unordered_map> labelToInsertJobs; // A mapping to hold invalid jobs, so we can dispose them upon index deletion. @@ -661,6 +659,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto svs_index = index->GetSVSIndex(); if (index->backendIndex->indexSize() == 0) { // No need to run GC on an empty index. + index->indexGCScheduled.clear(); return; } std::lock_guard lock(index->updateJobMutex); @@ -736,16 +735,11 @@ class TieredSVSIndex : public VecSimTieredIndex { } void scheduleSVSIndexConsolidate(labelType label) { - labels_to_consolidate_.push_back(label); + AsyncJob *new_consolidate_job = new (this->allocator) + SVSConsolidateJob(this->allocator, {label}, SVSIndexConsolidateWrapper, this); - if (labels_to_consolidate_.size() >= consolidateTriggerThreshold) { - AsyncJob *new_consolidate_job = new (this->allocator) - SVSConsolidateJob(this->allocator, labels_to_consolidate_, SVSIndexConsolidateWrapper, this); - - // Insert job to the queue. - this->submitSingleJob(new_consolidate_job); - labels_to_consolidate_.clear(); - } + // Insert job to the queue. + this->submitSingleJob(new_consolidate_job); } private: @@ -942,8 +936,6 @@ class TieredSVSIndex : public VecSimTieredIndex { ? SVS_VAMANA_DEFAULT_TRAINING_THRESHOLD : this->updateTriggerThreshold; - this->consolidateTriggerThreshold = SVS_VAMANA_DEFAULT_CONSOLIDATE_THRESHOLD; - this->trainingTriggerThreshold = tiered_svs_params.trainingTriggerThreshold == 0 ? default_training_threshold @@ -1016,18 +1008,20 @@ class TieredSVSIndex : public VecSimTieredIndex { if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { // Add vector to the frontend index. std::lock_guard lock(this->flatIndexGuard); - if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { - deleteAndUpdateInitIds(label); - } - ids_to_init_.insert(this->frontendIndex->indexSize()); - const auto ft_ret = this->frontendIndex->addVector(blob, label); - ret = std::max(ret + ft_ret, 0); + if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { + if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { + deleteAndUpdateInitIds(label); + } + ids_to_init_.insert(this->frontendIndex->indexSize()); + const auto ft_ret = this->frontendIndex->addVector(blob, label); + ret = std::max(ret + ft_ret, 0); - if (this->frontendIndex->indexSize() >= this->trainingTriggerThreshold) { - this->backendInitSubmited.store(true, std::memory_order_release); - scheduleSVSIndexInit(); + if (this->frontendIndex->indexSize() >= this->trainingTriggerThreshold) { + this->backendInitSubmited.store(true, std::memory_order_release); + scheduleSVSIndexInit(); + } + return ret; } - return ret; } this->flatIndexGuard.lock_shared(); diff --git a/tests/unit/test_svs_fp16.cpp b/tests/unit/test_svs_fp16.cpp index 95a26e6c2..dc36615f6 100644 --- a/tests/unit/test_svs_fp16.cpp +++ b/tests/unit/test_svs_fp16.cpp @@ -2857,8 +2857,9 @@ TYPED_TEST(FP16SVSTieredIndexTest, deleteVector) { ASSERT_EQ(tiered_index->indexSize(), 1); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 1); - // Move the vector to SVS by executing the insert job. - mock_thread_pool.thread_iteration(); + // Move the vector to SVS by executing the pending jobs. + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->indexLabelCount(), 1); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 1); // Scalar quantization accuracy is insufficient for this check. @@ -2905,7 +2906,9 @@ TYPED_TEST(FP16SVSTieredIndexTest, deleteVectorMulti) { ASSERT_EQ(tiered_index->deleteVector(vec_label), 2); ASSERT_EQ(tiered_index->indexSize(), 0); ASSERT_EQ(tiered_index->indexLabelCount(), 0); - mock_thread_pool.thread_iteration(); + + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); // Test deleting a label for which both of its vector's is in the flat index. diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 6ef887f79..bbf655667 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -1088,8 +1088,9 @@ TYPED_TEST(SVSTieredIndexTest, deleteVector) { ASSERT_EQ(tiered_index->indexSize(), 1); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 1); - // Move the vector to SVS by executing the insert job. - mock_thread_pool.thread_iteration(); + // Move the vector to SVS by executing the pending jobs. + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->indexLabelCount(), 1); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 1); // Scalar quantization accuracy is insufficient for this check. @@ -1218,6 +1219,8 @@ TYPED_TEST(SVSTieredIndexTestBasic, deleteVectorMulti) { ASSERT_EQ(tiered_index->deleteVector(vec_label), 2); ASSERT_EQ(tiered_index->indexSize(), 0); ASSERT_EQ(tiered_index->indexLabelCount(), 0); + + mock_thread_pool.thread_iteration(); mock_thread_pool.thread_iteration(); ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); @@ -2662,7 +2665,8 @@ TYPED_TEST(SVSTieredIndexTestBasic, overwriteVectorBasic) { ASSERT_EQ(tiered_index->getDistanceFrom_Unsafe(0, overwritten_vec), 0); // Ingest the updated vector to SVS. - mock_thread_pool.thread_iteration(); + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 1); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->indexLabelCount(), 1); @@ -3146,12 +3150,12 @@ TYPED_TEST(SVSTieredIndexTestBasic, runGCAPI) { // Run the GC API call, expect that we will clean up the SVS index. VecSimTieredIndex_GC(tiered_index); // Expected that a single GC job was added to the queue. - ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + 1); + ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + mock_thread_pool.thread_pool_size); // Run GC twice. VecSimTieredIndex_GC(tiered_index); - // Expected that no new GC job was added to the queue (indexGCScheduled is still set until the - // pending job runs). - ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + 1); + // Expected that no new GC jobs were added to the queue (indexGCScheduled is still set until the + // pending jobs run). + ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + mock_thread_pool.thread_pool_size); // Wait for any pending jobs to complete. As far as SVS GC is done via a job. mock_thread_pool.init_threads(); mock_thread_pool.thread_pool_join(); From db1d10616cd4dfc403da49b6adeee92a7e1a8708 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Fri, 21 Aug 2026 21:27:51 -0700 Subject: [PATCH 17/64] vecsim separate fin-grain index enabling --- src/VecSim/algorithms/svs/svs.h | 6 ++- src/VecSim/algorithms/svs/svs_extensions.h | 22 +++++++--- .../algorithms/svs/svs_serializer_impl.h | 6 +-- src/VecSim/algorithms/svs/svs_utils.h | 40 ++++++++++++++++--- 4 files changed, 57 insertions(+), 17 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 33cfbc334..663725d7c 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -93,10 +93,12 @@ class SVSIndex : public VecSimIndexAbstract, fl using graph_builder_t = SVSGraphBuilder; using graph_type = typename graph_builder_t::graph_type; + // The concurrent index mirrors the upstream template signature and member surface, so + // the only change needed here is the namespace. using impl_type = std::conditional_t< isMulti, - svs::index::vamana::MultiMutableVamanaIndex, - svs::index::vamana::MutableVamanaIndex>; + svs::concurrent::MultiMutableVamanaIndex, + svs::concurrent::MutableVamanaIndex>; bool forcePreprocessing; diff --git a/src/VecSim/algorithms/svs/svs_extensions.h b/src/VecSim/algorithms/svs/svs_extensions.h index 3903d2289..1700b30c9 100644 --- a/src/VecSim/algorithms/svs/svs_extensions.h +++ b/src/VecSim/algorithms/svs/svs_extensions.h @@ -10,10 +10,20 @@ #pragma once #include "VecSim/algorithms/svs/svs_utils.h" #include "svs/extensions/vamana/scalar.h" +// Tells SQDataset that the concurrent SegmentedBlocked tag is a blocked allocator, so +// that resize()/compact() are not constrained away. Without it a scalar-quantized +// concurrent index fails to compile the moment the index grows. +#include "svs/concurrent/extensions/scalar.h" #if HAVE_SVS_LVQ #include SVS_LVQ_HEADER #include SVS_LEANVEC_HEADER +// The same treatment for the LVQ and LeanVec datasets, each of which keeps its own private +// copy of the blocked-allocator trait. LeanVec additionally picks the allocator for its inner +// datasets through that trait, and its blocked branch hardcodes svs::data::Blocked -- so +// without this the inner storage would quietly lose the grow-stable guarantee. +#include "svs/concurrent/extensions/lvq.h" +#include "svs/concurrent/extensions/leanvec.h" #endif // HAVE_SVS_LVQ // Scalar Quantization traits for SVS @@ -21,7 +31,7 @@ template struct SVSStorageTraits { using element_type = std::int8_t; using allocator_type = svs_details::SVSAllocator; - using blocked_type = svs::data::Blocked>; + using blocked_type = svs::concurrent::SegmentedBlocked>; using index_storage_type = svs::quantization::scalar::SQDataset; @@ -32,7 +42,7 @@ struct SVSStorageTraits { // SVS block size is a power of two, so we can use it directly auto svs_bs = svs_details::SVSBlockSize(block_size, element_size(dim)); allocator_type data_allocator{std::move(allocator)}; - return svs::make_blocked_allocator_handle({svs_bs}, data_allocator); + return svs_details::make_segmented_blocked_allocator_handle({svs_bs}, data_allocator); } static constexpr VecSimSvsQuantBits get_compression_mode() { return VecSimSvsQuant_Scalar; } @@ -89,7 +99,7 @@ template struct SVSStorageTraits 1)>> { using allocator_type = svs_details::SVSAllocator; - using blocked_type = svs::data::Blocked>; + using blocked_type = svs::concurrent::SegmentedBlocked>; using strategy_type = typename svs_details::LVQSelector::strategy; using index_storage_type = svs::quantization::lvq::LVQDataset @@ -163,7 +173,7 @@ struct SVSStorageTraits struct SVSStorageTraits { using allocator_type = svs_details::SVSAllocator; - using blocked_type = svs::data::Blocked>; + using blocked_type = svs::concurrent::SegmentedBlocked>; using index_storage_type = svs::leanvec::LeanDataset, svs::leanvec::UsingLVQ, svs::Dynamic, svs::Dynamic, blocked_type>; @@ -193,7 +203,7 @@ struct SVSStorageTraits { // SVS block size is a power of two, so we can use it directly auto svs_bs = svs_details::SVSBlockSize(block_size, element_size(dim)); allocator_type data_allocator{std::move(allocator)}; - return svs::make_blocked_allocator_handle({svs_bs}, data_allocator); + return svs_details::make_segmented_blocked_allocator_handle({svs_bs}, data_allocator); } template diff --git a/src/VecSim/algorithms/svs/svs_serializer_impl.h b/src/VecSim/algorithms/svs/svs_serializer_impl.h index 6f6757922..5e551ef81 100644 --- a/src/VecSim/algorithms/svs/svs_serializer_impl.h +++ b/src/VecSim/algorithms/svs/svs_serializer_impl.h @@ -93,17 +93,17 @@ void SVSIndex compareMetadataFile(folder_path + "/metadata"); if constexpr (isMulti) { - auto loaded = svs::index::vamana::auto_multi_dynamic_assemble( + auto loaded = svs::concurrent::auto_multi_dynamic_assemble( folder_path + "/config", SVS_LAZY(graph_builder_t::load(folder_path + "/graph", this->blockSize, this->buildParams, this->getAllocator())), SVS_LAZY(storage_traits_t::load(folder_path + "/data", this->blockSize, this->dim, this->getAllocator())), distance_f(), std::move(threadpool_handle), - svs::index::vamana::MultiMutableVamanaLoad::FROM_MULTI, logger_); + svs::concurrent::MultiMutableVamanaLoad::FROM_MULTI, logger_); impl_ = std::make_unique(std::move(loaded)); } else { - auto loaded = svs::index::vamana::auto_dynamic_assemble( + auto loaded = svs::concurrent::auto_dynamic_assemble( folder_path + "/config", SVS_LAZY(graph_builder_t::load(folder_path + "/graph", this->blockSize, this->buildParams, this->getAllocator())), diff --git a/src/VecSim/algorithms/svs/svs_utils.h b/src/VecSim/algorithms/svs/svs_utils.h index 2e240358f..3c1de0f14 100644 --- a/src/VecSim/algorithms/svs/svs_utils.h +++ b/src/VecSim/algorithms/svs/svs_utils.h @@ -14,6 +14,10 @@ #include "svs/core/distance.h" #include "svs/lib/float16.h" #include "svs/index/vamana/dynamic_index.h" +// Pulls in `svs::index::vamana::concurrent` and the `svs::concurrent` alias. The concurrent +// namespace redeclares only the entities it has to replace; every other name (build/search +// parameters, extension points) still resolves to the enclosing `svs::index::vamana`. +#include "svs/concurrent/concurrent.h" #if HAVE_SVS_LVQ #include "svs/cpuid.h" @@ -198,6 +202,21 @@ inline std::pair isSVSQuantBitsSupported(VecSimSvsQuan // unreachable code, but to avoid compiler warning return std::make_pair(VecSimSvsQuant_NONE, false); } + +// The `svs::concurrent` counterpart of `svs::make_blocked_allocator_handle`. +// +// The upstream helper cannot be reused here: it hardcodes `svs::data::Blocked` in its return +// type, so there is no way to ask it for the grow-stable tag. The type-erasing +// `AllocatorHandle` wrapper is what makes the compressed datasets (which take an allocator +// handle rather than a concrete allocator) work with VecSim's own allocator. +template +svs::concurrent::SegmentedBlocked> +make_segmented_blocked_allocator_handle(const svs::data::BlockingParameters ¶meters, + Alloc alloc) { + using handle_type = svs::AllocatorHandle; + return svs::concurrent::SegmentedBlocked{ + parameters, svs::make_allocator_handle(std::move(alloc))}; +} } // namespace svs_details template ; // Used in creating storage + // `SegmentedBlocked` is a drop-in replacement for `svs::data::Blocked` (it derives from + // it and carries the same blocking parameters) that additionally keeps already-published + // elements at a stable address when the dataset grows, which is what allows a concurrent + // reader to keep a pointer into the data while a writer appends. + using blocked_type = svs::concurrent::SegmentedBlocked; // svs::Dynamic means runtime dimensionality in opposite to compile-time dimensionality - using index_storage_type = svs::data::BlockedData; + using index_storage_type = + svs::concurrent::SegmentedBlockedData; static constexpr bool is_compressed() { return false; } @@ -272,9 +296,13 @@ struct SVSStorageTraits { template struct SVSGraphBuilder { using allocator_type = svs_details::SVSAllocator; - using blocked_type = svs::data::Blocked; - using graph_data_type = svs::data::BlockedData; - using graph_type = svs::graphs::SimpleGraph; + using blocked_type = svs::concurrent::SegmentedBlocked; + using graph_data_type = + svs::concurrent::SegmentedBlockedData; + // The concurrent `SimpleGraph` has the same shape as the upstream one but guards each + // adjacency list with a sequence lock, so a reader can traverse an edge list that a + // writer is concurrently rewriting. + using graph_type = svs::concurrent::graphs::SimpleGraph; static blocked_type make_blocked_allocator(size_t block_size, size_t graph_max_degree, std::shared_ptr allocator) { @@ -304,7 +332,7 @@ struct SVSGraphBuilder { // based on the data types, which we found to perform better through heuristic analysis. auto prefetch_parameters = svs::index::vamana::extensions::estimate_prefetch_parameters(data); - auto builder = svs::index::vamana::VamanaBuilder( + auto builder = svs::concurrent::VamanaBuilder( graph, data, std::move(distance), parameters, threadpool, prefetch_parameters, logger); // Specific to the Vamana algorithm: From a540c0a1227a4514ad26b7ffce2272e8cc41152a Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Wed, 26 Aug 2026 21:06:45 -0700 Subject: [PATCH 18/64] Rename SVSIndexBase thread API to match upstream Upstream renamed the three thread-control methods on SVSIndexBase: getNumThreads -> getParallelism setNumThreads -> setParallelism getThreadPoolCapacity -> getPoolSize Adopt those names here ahead of merging upstream/main. This is a pure rename -- 32 lines across 6 files, mechanically verified by reversing the substitution and diffing against the parent commit. Method bodies deliberately keep this branch's threadpool API (size/resize/capacity), since VecSimSVSThreadPool here is still the per-index owned pool. Upstream reworked it into a process-wide singleton with thread renting, sized via VecSim_UpdateThreadPoolSize(); that is genuine divergence to reconcile in the merge, not something a rename should paper over. The point is to remove this conflict class before merging. The names collided on roughly half the affected lines without producing conflict markers, so git resolved some toward upstream and some toward here, yielding a tree that referenced methods it no longer declared. Co-Authored-By: Claude Opus 5 --- src/VecSim/algorithms/svs/svs.h | 20 ++++++++++---------- src/VecSim/algorithms/svs/svs_tiered.h | 26 +++++++++++++------------- tests/benchmark/bm_utils.h | 4 ++-- tests/benchmark/bm_vecsim_svs.h | 2 +- tests/unit/test_svs_fp16.cpp | 6 +++--- tests/unit/test_svs_tiered.cpp | 6 +++--- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 663725d7c..b2a7699ba 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -44,9 +44,9 @@ struct SVSIndexBase virtual void consolidate(const std::vector& labels) = 0; virtual bool isLabelExists(labelType label) const = 0; virtual size_t indexStorageSize() const = 0; - virtual size_t getNumThreads() const = 0; - virtual void setNumThreads(size_t numThreads) = 0; - virtual size_t getThreadPoolCapacity() const = 0; + virtual size_t getParallelism() const = 0; + virtual void setParallelism(size_t parallelism) = 0; + virtual size_t getPoolSize() const = 0; virtual bool isCompressed() const = 0; virtual bool ready() const = 0; @@ -504,8 +504,8 @@ class SVSIndex : public VecSimIndexAbstract, fl .maxCandidatePoolSize = this->buildParams.max_candidate_pool_size, .pruneTo = this->buildParams.prune_to, .useSearchHistory = this->buildParams.use_full_search_history, - .numThreads = this->getThreadPoolCapacity(), - .lastReservedThreads = this->getNumThreads(), + .numThreads = this->getPoolSize(), + .lastReservedThreads = this->getParallelism(), .numberOfMarkedDeletedNodes = this->num_marked_deleted, .searchWindowSize = this->search_window_size, .searchBufferCapacity = this->search_buffer_capacity, @@ -611,14 +611,14 @@ class SVSIndex : public VecSimIndexAbstract, fl // Enforce single-threaded execution for single vector operations to ensure optimal // performance and consistent behavior. Callers must set numThreads=1 before calling this // method. - assert(getNumThreads() == 1 && "Can't use more than one thread to insert a single vector"); + assert(getParallelism() == 1 && "Can't use more than one thread to insert a single vector"); return addVectorsImpl(vector_data, &label, 1); } int addVectors(const void *vectors_data, const labelType *labels, size_t n) override { // Prevent misuse: single vector operations should use addVector(), not addVectors() with // n=1 This ensures proper thread management and API contract enforcement. - assert(!(n == 1 && getNumThreads() > 1) && + assert(!(n == 1 && getParallelism() > 1) && "Can't use more than one thread to insert a single vector"); return addVectorsImpl(vectors_data, labels, n); } @@ -638,10 +638,10 @@ class SVSIndex : public VecSimIndexAbstract, fl } } - size_t getNumThreads() const override { return threadpool_.size(); } - void setNumThreads(size_t numThreads) override { threadpool_.resize(numThreads); } + size_t getParallelism() const override { return threadpool_.size(); } + void setParallelism(size_t parallelism) override { threadpool_.resize(parallelism); } - size_t getThreadPoolCapacity() const override { return threadpool_.capacity(); } + size_t getPoolSize() const override { return threadpool_.capacity(); } bool isCompressed() const override { return storage_traits_t::is_compressed(); } diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 27d2f4d12..a240fcd09 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -636,8 +636,8 @@ class TieredSVSIndex : public VecSimTieredIndex { // // No need to run GC on an empty index. // return; // } - // // svs_index->setNumThreads(std::min(availableThreads, index->backendIndex->indexSize())); - // svs_index->setNumThreads(1); + // // svs_index->setParallelism(std::min(availableThreads, index->backendIndex->indexSize())); + // svs_index->setParallelism(1); // // VecSimIndexAbstract::runGC() is protected // fprintf(stderr, "runGC\n"); // static_cast(index->backendIndex)->runGC(); @@ -664,10 +664,10 @@ class TieredSVSIndex : public VecSimTieredIndex { } std::lock_guard lock(index->updateJobMutex); - svs_index->setNumThreads(std::min(availableThreads, index->backendIndex->indexSize())); + svs_index->setParallelism(std::min(availableThreads, index->backendIndex->indexSize())); // VecSimIndexAbstract::runGC() is protected static_cast(index->backendIndex)->runGC(); - svs_index->setNumThreads(1); + svs_index->setParallelism(1); // Release the scheduled flag to allow scheduling again index->indexGCScheduled.clear(); @@ -679,7 +679,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::shared_lock lock(index->updateJobMutex); auto svs_index = index->GetSVSIndex(); - svs_index->setNumThreads(1); + svs_index->setParallelism(1); svs_index->consolidate(consolidate_job->labels); delete job; } @@ -708,7 +708,7 @@ class TieredSVSIndex : public VecSimTieredIndex { return; } - auto total_threads = this->GetSVSIndex()->getThreadPoolCapacity(); + auto total_threads = this->GetSVSIndex()->getPoolSize(); auto jobs = SVSMultiThreadJob::createJobs( this->allocator, SVS_BATCH_UPDATE_JOB, initSVSIndexWrapper, this, total_threads, std::chrono::microseconds(updateJobWaitTime), &uncompletedJobs); @@ -721,7 +721,7 @@ class TieredSVSIndex : public VecSimTieredIndex { return; } - auto total_threads = this->GetSVSIndex()->getThreadPoolCapacity(); + auto total_threads = this->GetSVSIndex()->getPoolSize(); auto jobs = SVSMultiThreadJob::createJobs( this->allocator, SVS_GC_JOB, SVSIndexGCWrapper, this, total_threads, std::chrono::microseconds(updateJobWaitTime), &uncompletedJobs); @@ -805,7 +805,7 @@ class TieredSVSIndex : public VecSimTieredIndex { memcpy(blob_copy.get(), this->frontendIndex->getDataByInternalId(job->id), data_size); this->flatIndexGuard.unlock_shared(); - // svs_index->setNumThreads(1); + // svs_index->setParallelism(1); { std::shared_lock lock(updateJobMutex); svs_index->addVector(blob_copy.get(), job->label); @@ -876,11 +876,11 @@ class TieredSVSIndex : public VecSimTieredIndex { { auto svs_index = GetSVSIndex(); - svs_index->setNumThreads(std::min(availableThreads, labels_to_move.size())); + svs_index->setParallelism(std::min(availableThreads, labels_to_move.size())); assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), labels_to_move.size()); - svs_index->setNumThreads(1); + svs_index->setParallelism(1); svs_index->setImpl(std::move(impl)); } @@ -990,9 +990,9 @@ class TieredSVSIndex : public VecSimTieredIndex { std::lock_guard lock(this->updateJobMutex); // Set available thread count to 1 for single vector write-in-place operation. // This maintains the contract that single vector operations use exactly one thread. - // TODO: Replace this setNumThreads(1) call with an assertion once we establish + // TODO: Replace this setParallelism(1) call with an assertion once we establish // a contract that write-in-place mode guarantees numThreads == 1. - svs_index->setNumThreads(1); + svs_index->setParallelism(1); int deleted = 0; if (!this->backendIndex->isMultiValue()) { deleted = svs_index->deleteVector(label); @@ -1392,7 +1392,7 @@ class TieredSVSIndex : public VecSimTieredIndex { return; } // Force single thread for write-in-place mode. - this->GetSVSIndex()->setNumThreads(1); + this->GetSVSIndex()->setParallelism(1); // VecSimIndexAbstract::runGC() is protected static_cast(this->backendIndex)->runGC(); return; diff --git a/tests/benchmark/bm_utils.h b/tests/benchmark/bm_utils.h index 88f3e2e5d..e952794ad 100644 --- a/tests/benchmark/bm_utils.h +++ b/tests/benchmark/bm_utils.h @@ -30,9 +30,9 @@ CreateTieredSVSParams(VecSimParams &svs_params, tieredIndexMock &mock_thread_poo template static void verifyNumThreads(TieredSVSIndex *tiered_index, size_t expected_num_threads, size_t expected_capcity, std::string msg = "") { - ASSERT_EQ(tiered_index->GetSVSIndex()->getThreadPoolCapacity(), expected_capcity) + ASSERT_EQ(tiered_index->GetSVSIndex()->getPoolSize(), expected_capcity) << msg << ": thread pool capacity mismatch"; - size_t num_reserved_threads = tiered_index->GetSVSIndex()->getNumThreads(); + size_t num_reserved_threads = tiered_index->GetSVSIndex()->getParallelism(); if (num_reserved_threads < expected_num_threads) { std::cout << msg << ": WARNING: last reserved threads (" << num_reserved_threads << ") is less than expected (" << expected_num_threads << ")." << std::endl; diff --git a/tests/benchmark/bm_vecsim_svs.h b/tests/benchmark/bm_vecsim_svs.h index fe6c92493..c4b167421 100644 --- a/tests/benchmark/bm_vecsim_svs.h +++ b/tests/benchmark/bm_vecsim_svs.h @@ -115,7 +115,7 @@ class BM_VecSimSVS : public BM_VecSimGeneral { tiered_params.primaryIndexParams->algoParams.svsParams.num_threads; size_t num_threads = params_threadpool_size ? params_threadpool_size : mock_thread_pool.thread_pool_size; - tiered_index->GetSVSIndex()->setNumThreads(num_threads); + tiered_index->GetSVSIndex()->setParallelism(num_threads); test_utils::verifyNumThreads(tiered_index, num_threads, num_threads, std::string("CreateTieredSVSIndex")); diff --git a/tests/unit/test_svs_fp16.cpp b/tests/unit/test_svs_fp16.cpp index dc36615f6..f1746fa13 100644 --- a/tests/unit/test_svs_fp16.cpp +++ b/tests/unit/test_svs_fp16.cpp @@ -2242,8 +2242,8 @@ class FP16SVSTieredIndexTest : public FP16SVSTest { } void verifyNumThreads(TieredSVSIndex *tiered_index, size_t expected_num_threads, size_t expected_capcity) { - ASSERT_EQ(tiered_index->GetSVSIndex()->getNumThreads(), expected_num_threads); - ASSERT_EQ(tiered_index->GetSVSIndex()->getThreadPoolCapacity(), expected_capcity); + ASSERT_EQ(tiered_index->GetSVSIndex()->getParallelism(), expected_num_threads); + ASSERT_EQ(tiered_index->GetSVSIndex()->getPoolSize(), expected_capcity); } TieredSVSIndex *CreateTieredSVSIndex(const TieredIndexParams &tiered_params, @@ -2258,7 +2258,7 @@ class FP16SVSTieredIndexTest : public FP16SVSTest { // Set number of available threads to 1 unless specified otherwise, // so we can insert one vector at a time directly to svs. - tiered_index->GetSVSIndex()->setNumThreads(num_available_threads); + tiered_index->GetSVSIndex()->setParallelism(num_available_threads); size_t params_threadpool_size = tiered_params.primaryIndexParams->algoParams.svsParams.num_threads; size_t expected_capacity = diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index bbf655667..e701489d6 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -92,8 +92,8 @@ class SVSTieredIndexTest : public ::testing::Test { void verifyNumThreads(TieredSVSIndex *tiered_index, size_t expected_num_threads, size_t expected_capcity) { - ASSERT_EQ(tiered_index->GetSVSIndex()->getNumThreads(), expected_num_threads); - ASSERT_EQ(tiered_index->GetSVSIndex()->getThreadPoolCapacity(), expected_capcity); + ASSERT_EQ(tiered_index->GetSVSIndex()->getParallelism(), expected_num_threads); + ASSERT_EQ(tiered_index->GetSVSIndex()->getPoolSize(), expected_capcity); } TieredSVSIndex *CreateTieredSVSIndex(const TieredIndexParams &tiered_params, tieredIndexMock &mock_thread_pool, @@ -108,7 +108,7 @@ class SVSTieredIndexTest : public ::testing::Test { // which requires exactly 1 thread. When using tiered index addVector API, // the thread count is managed internally according to the operation and threadpool // capacity, so testing parallelism remains intact. - tiered_index->GetSVSIndex()->setNumThreads(num_available_threads); + tiered_index->GetSVSIndex()->setParallelism(num_available_threads); size_t params_threadpool_size = tiered_params.primaryIndexParams->algoParams.svsParams.num_threads; size_t expected_capacity = From ac97868f6a3ac69f75148111e8308747c8a7d1e2 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Tue, 1 Sep 2026 21:08:18 -0700 Subject: [PATCH 19/64] CI fixes --- src/VecSim/algorithms/svs/svs.h | 2 +- tests/unit/test_svs_tiered.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 9cf7d494f..b41e7ecdd 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -323,7 +323,7 @@ class SVSIndex : public VecSimIndexAbstract, fl return n - deleted_num; } - void consolidate(const std::vector& labels) { + void consolidate(const std::vector& labels) override { std::shared_lock lock(this->pimplGuard_); if (!ready()) return; diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index e7142f455..b1675b6ca 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -519,7 +519,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, ShrinkDuringScheduledUpdateIsDeferred) { shrink_callback_ran = true; }); - tiered_index->scheduleSVSIndexUpdate(); + tiered_index->scheduleSVSIndexInit(); ASSERT_EQ(mock_thread_pool.jobQ.size(), num_threads); mock_thread_pool.init_threads(); From a32aa9d3cd5e16b1585e907f86b7cff963c88e3c Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 2 Sep 2026 23:51:44 -0700 Subject: [PATCH 20/64] clean-up commented and unused dead code --- src/VecSim/algorithms/svs/svs_tiered.h | 34 -------------------------- src/VecSim/vec_sim_common.h | 1 - 2 files changed, 35 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 7bdebb0c0..8e03ac3e4 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -640,33 +640,6 @@ class TieredSVSIndex : public VecSimTieredIndex { * @note no need to implement extra non-static method, as GC logic is simple enough to be done * here. */ - // static void SVSIndexGCWrapper(AsyncJob *job) { - // auto gc_job = static_cast(job); - // auto index = static_cast *>(gc_job->index); - - // std::shared_lock lock(index->updateJobMutex); - // // Do SVS index GC - // index->backendIndex->log(VecSimCommonStrings::LOG_VERBOSE_STRING, - // "running asynchronous GC for tiered SVS index"); - // auto svs_index = index->GetSVSIndex(); - // if (index->backendIndex->indexSize() == 0) { - // index->indexGCScheduled.clear(); - // delete job; - // // No need to run GC on an empty index. - // return; - // } - // // svs_index->setParallelism(std::min(availableThreads, index->backendIndex->indexSize())); - // svs_index->setParallelism(1); - // // VecSimIndexAbstract::runGC() is protected - // fprintf(stderr, "runGC\n"); - // static_cast(index->backendIndex)->runGC(); - // fprintf(stderr, "GC done\n"); - - // // Release the scheduled flag to allow scheduling again - // index->indexGCScheduled.clear(); - // delete job; - // } - static void SVSIndexGCWrapper(VecSimIndex *idx, size_t availableThreads) { assert(availableThreads > 0); auto index = static_cast *>(idx); @@ -744,12 +717,6 @@ class TieredSVSIndex : public VecSimTieredIndex { this->allocator, SVS_GC_JOB, SVSIndexGCWrapper, this, std::chrono::microseconds(updateJobWaitTime), &uncompletedJobs); this->submitJobs(jobs); - - // AsyncJob *new_GC_job = new (this->allocator) - // SVSGCJob(this->allocator, SVSIndexGCWrapper, this); - - // // Insert job to the queue. - // this->submitSingleJob(new_GC_job); } void scheduleSVSIndexConsolidate(labelType label) { @@ -823,7 +790,6 @@ class TieredSVSIndex : public VecSimTieredIndex { memcpy(blob_copy.get(), this->frontendIndex->getDataByInternalId(job->id), data_size); this->flatIndexGuard.unlock_shared(); - // svs_index->setParallelism(1); { std::shared_lock lock(updateJobMutex); svs_index->addVector(blob_copy.get(), job->label); diff --git a/src/VecSim/vec_sim_common.h b/src/VecSim/vec_sim_common.h index b3ded66c0..0f1a360cb 100644 --- a/src/VecSim/vec_sim_common.h +++ b/src/VecSim/vec_sim_common.h @@ -50,7 +50,6 @@ extern "C" { #define SVS_VAMANA_DEFAULT_TRAINING_THRESHOLD (10 * DEFAULT_BLOCK_SIZE) // 10 * 1024 vectors // Default batch update threshold for SVS index. #define SVS_VAMANA_DEFAULT_UPDATE_THRESHOLD (1 * DEFAULT_BLOCK_SIZE) // 1 * 1024 vectors -#define SVS_VAMANA_DEFAULT_CONSOLIDATE_THRESHOLD (1 * DEFAULT_BLOCK_SIZE) // 1 * 1024 vectors #define SVS_VAMANA_DEFAULT_SEARCH_WINDOW_SIZE 10 // NOTE: No need to have SVS_VAMANA_DEFAULT_SEARCH_BUFFER_CAPACITY // as the default is determined by the search_window_size From 978a49e778b5511581d22aa0a1a2a502388f88e9 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 3 Sep 2026 06:04:36 -0700 Subject: [PATCH 21/64] harmonize test --- tests/unit/test_svs_tiered.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index b1675b6ca..061ff75ec 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -603,7 +603,9 @@ TYPED_TEST(SVSTieredIndexTestBasic, ShrinkDuringScheduledGCIsDeferred) { }); VecSimTieredIndex_GC(tiered_index); - ASSERT_EQ(mock_thread_pool.jobQ.size(), num_threads); + // Each soft-deleted slot has corresponding consolidate() job + size_t num_consolidate = tiered_index->GetSVSIndex()->getNumMarkedDeleted(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), num_threads + num_consolidate); mock_thread_pool.init_threads(); mock_thread_pool.thread_pool_join(); From 2a06ff7e9f79bb8af061745ed8af4e3d86332cd2 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Fri, 4 Sep 2026 13:34:58 -0700 Subject: [PATCH 22/64] minor revisions for CI --- src/VecSim/algorithms/svs/svs_tiered.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 8e03ac3e4..72a8cb357 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -871,7 +871,7 @@ class TieredSVSIndex : public VecSimTieredIndex { } std::sort(ids_to_move.begin(), ids_to_move.end()); - int total_deleted = 0; + [[maybe_unused]] int total_deleted = 0; for (auto it = ids_to_move.rbegin(); it != ids_to_move.rend(); ++it) { idType id = *it; auto label = this->frontendIndex->getVectorLabel(id); @@ -1308,7 +1308,7 @@ class TieredSVSIndex : public VecSimTieredIndex { return main_results; } - return merge_result_lists(main_results, flat_results, k); + return merge_result_lists(main_results, flat_results, k); } } @@ -1345,12 +1345,12 @@ class TieredSVSIndex : public VecSimTieredIndex { sort_results_by_score_then_id(flat_results); auto code = main_results->code; - VecSimQueryReply *ret = merge_result_lists(main_results, flat_results, -1); + VecSimQueryReply *ret = merge_result_lists(main_results, flat_results, -1); ret->code = code; return ret; } else { // BY_ID concat_results(main_results, flat_results); - filter_results_by_id(main_results); + filter_results_by_id(main_results); return main_results; } } From 095ae3b4c533766ca04ee598306e1045b158946e Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 7 Sep 2026 01:27:01 -0700 Subject: [PATCH 23/64] fix build and tests --- src/VecSim/algorithms/svs/svs_tiered.h | 21 ++++++++++++++------- src/VecSim/algorithms/svs/svs_utils.h | 7 +++++++ src/VecSim/index_factories/svs_factory.cpp | 4 +++- tests/unit/test_svs.cpp | 16 +++++++++++++--- tests/unit/test_svs_fp16.cpp | 21 ++++++++++++++++----- tests/unit/test_svs_multi.cpp | 10 +++++++++- tests/unit/test_svs_tiered.cpp | 10 ++++++++-- 7 files changed, 70 insertions(+), 19 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 8e03ac3e4..a55c8185a 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -864,10 +864,17 @@ class TieredSVSIndex : public VecSimTieredIndex { auto svs_index = GetSVSIndex(); svs_index->setParallelism(std::min(availableThreads, labels_to_move.size())); assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); - auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), - labels_to_move.size()); - svs_index->setParallelism(1); - svs_index->setImpl(std::move(impl)); + // The backend may already have been initialized while this job was queued + if (svs_index->ready()) { + svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), + labels_to_move.size()); + svs_index->setParallelism(1); + } else { + auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), + labels_to_move.size()); + svs_index->setParallelism(1); + svs_index->setImpl(std::move(impl)); + } } std::sort(ids_to_move.begin(), ids_to_move.end()); @@ -1308,7 +1315,7 @@ class TieredSVSIndex : public VecSimTieredIndex { return main_results; } - return merge_result_lists(main_results, flat_results, k); + return merge_result_lists(main_results, flat_results, k); } } @@ -1345,12 +1352,12 @@ class TieredSVSIndex : public VecSimTieredIndex { sort_results_by_score_then_id(flat_results); auto code = main_results->code; - VecSimQueryReply *ret = merge_result_lists(main_results, flat_results, -1); + VecSimQueryReply *ret = merge_result_lists(main_results, flat_results, -1); ret->code = code; return ret; } else { // BY_ID concat_results(main_results, flat_results); - filter_results_by_id(main_results); + filter_results_by_id(main_results); return main_results; } } diff --git a/src/VecSim/algorithms/svs/svs_utils.h b/src/VecSim/algorithms/svs/svs_utils.h index 33c1135a9..45860c9ba 100644 --- a/src/VecSim/algorithms/svs/svs_utils.h +++ b/src/VecSim/algorithms/svs/svs_utils.h @@ -373,6 +373,13 @@ struct SVSGraphBuilder { // graph_max_degree neighbors return sizeof(SVSIdType) * (graph_max_degree + 1); } + + // Per-node size of the concurrent graph's reverse-edge index. + // + static constexpr size_t reverse_edges_element_size() { + using reverse_edges_type = typename graph_type::reverse_edges_type; + return sizeof(typename reverse_edges_type::list_type) + sizeof(svs::concurrent::SpinLock); + } }; // A slot in the shared SVS thread pool. Wraps an SVS Thread with an occupancy flag diff --git a/src/VecSim/index_factories/svs_factory.cpp b/src/VecSim/index_factories/svs_factory.cpp index 33adf6fb1..bae7c021d 100644 --- a/src/VecSim/index_factories/svs_factory.cpp +++ b/src/VecSim/index_factories/svs_factory.cpp @@ -234,10 +234,12 @@ size_t EstimateElementSize(const SVSParams *params) { // Assuming that the graph_max_degree can be unset in params. const auto graph_max_degree = svs_details::makeVamanaBuildParameters(*params).graph_max_degree; const auto graph_node_size = SVSGraphBuilder::element_size(graph_max_degree); + const auto reverse_edges_node_size = + SVSGraphBuilder::reverse_edges_element_size(); const auto vector_size = QuantizedVectorSize(params->type, params->quantBits, params->dim, 0, params->leanvec_dim); - return vector_size + graph_node_size; + return vector_size + graph_node_size + reverse_edges_node_size; } size_t EstimateInitialSize(const SVSParams *params, bool is_normalized) { diff --git a/tests/unit/test_svs.cpp b/tests/unit/test_svs.cpp index 27b381327..3105372b2 100644 --- a/tests/unit/test_svs.cpp +++ b/tests/unit/test_svs.cpp @@ -35,6 +35,16 @@ } \ } +// Expected allocation for one block of `block_size` element slots, of which only +// `num_elements` are actually populated. +// +static size_t EstimateBlockSize(const SVSParams ¶ms, size_t block_size, size_t num_elements) { + const size_t reverse_edges_per_slot = SVSGraphBuilder::reverse_edges_element_size(); + const size_t reverse_edges_slots = svs::lib::SegmentedVector(num_elements).capacity(); + return (EstimateElementSize(params) - reverse_edges_per_slot) * block_size + + reverse_edges_per_slot * reverse_edges_slots; +} + // Log callback function to print non-debug log messages static void svsTestLogCallBackNoDebug(void *ctx, const char *level, const char *message) { if (level == nullptr || message == nullptr) { @@ -1996,7 +2006,7 @@ TYPED_TEST(SVSTest, testSizeEstimation) { size_t actual = index->getAllocationSize(); ASSERT_EQ(estimation, actual); - estimation = EstimateElementSize(params) * bs; + estimation = EstimateBlockSize(params, bs, 1); GenerateAndAddVector(index, dim, 0); actual = index->getAllocationSize() - actual; // get the delta @@ -2857,7 +2867,7 @@ TEST(SVSTest, quant_modes) { ASSERT_EQ(VecSimIndex_IndexSize(index), n); - estimation = EstimateElementSize(params) * params.blockSize; + estimation = EstimateBlockSize(params, params.blockSize, n); actual = index->getAllocationSize() - actual; // get the delta ASSERT_GT(actual, 0); // LVQ element size estimation accuracy is low @@ -3219,7 +3229,7 @@ TEST(SVSTest, scalar_quantization_query) { ASSERT_EQ(VecSimIndex_IndexSize(index_sq), n); ASSERT_EQ(index_sq->indexCapacity(), n); - estimation = EstimateElementSize(params) * params.blockSize; + estimation = EstimateBlockSize(params, params.blockSize, n); actual = index_sq->getAllocationSize() - actual; // get the delta ASSERT_GT(actual, 0); ASSERT_GE(estimation * 1.01, actual); diff --git a/tests/unit/test_svs_fp16.cpp b/tests/unit/test_svs_fp16.cpp index 895e8f8a5..63ccf1aab 100644 --- a/tests/unit/test_svs_fp16.cpp +++ b/tests/unit/test_svs_fp16.cpp @@ -40,6 +40,13 @@ using float16 = vecsim_types::float16; } \ } +static size_t EstimateBlockSize(const SVSParams ¶ms, size_t block_size, size_t num_elements) { + const size_t reverse_edges_per_slot = SVSGraphBuilder::reverse_edges_element_size(); + const size_t reverse_edges_slots = svs::lib::SegmentedVector(num_elements).capacity(); + return (EstimateElementSize(params) - reverse_edges_per_slot) * block_size + + reverse_edges_per_slot * reverse_edges_slots; +} + // Log callback function to print non-debug log messages static void svsTestLogCallBackNoDebug(void *ctx, const char *level, const char *message) { if (level == nullptr || message == nullptr) { @@ -1388,7 +1395,7 @@ TYPED_TEST(FP16SVSTest, testSizeEstimation) { size_t actual = index->getAllocationSize(); ASSERT_EQ(estimation, actual); - estimation = EstimateElementSize(params) * bs; + estimation = EstimateBlockSize(params, bs, 1); this->GenerateAndAddVector(index, dim, 0); actual = index->getAllocationSize() - actual; // get the delta @@ -1589,7 +1596,7 @@ TYPED_TEST(FP16SVSTest, quant_modes) { ASSERT_EQ(VecSimIndex_IndexSize(index), n); - estimation = EstimateElementSize(params) * DEFAULT_BLOCK_SIZE; + estimation = EstimateBlockSize(params, DEFAULT_BLOCK_SIZE, n); actual = index->getAllocationSize() - actual; // get the delta ASSERT_GT(actual, 0); // LVQ element size estimation accuracy is low @@ -1666,15 +1673,19 @@ TYPED_TEST(FP16SVSTest, scalar_quantization_query) { const size_t k = 10; const double quant_precision = 1.0 / (1 << 7); // int8 quantization precision + std::default_random_engine gen; + std::uniform_real_distribution dist(-1.0, 1.0); std::vector> dataset(n); for (size_t i = 0; i < n; i++) { - test_utils::populate_float16_vec(dataset[i].data(), dim, i, -1.0, 1.0); + for (size_t j = 0; j < dim; j++) { + dataset[i][j] = vecsim_types::FP32_to_FP16(dist(gen)); + } } std::vector ids(n); std::iota(ids.begin(), ids.end(), 0); float16 query[dim]; - test_utils::populate_float16_vec(query, dim, n, -1.0, 1.0); + this->GenerateVector(query, dim, 0.1f); VecSimQueryReply *fp_results = nullptr; auto verify_res = [&](size_t id, double score, size_t result_rank) { @@ -1724,7 +1735,7 @@ TYPED_TEST(FP16SVSTest, scalar_quantization_query) { ASSERT_EQ(VecSimIndex_IndexSize(index_sq), n); ASSERT_EQ(index_sq->indexCapacity(), n); - estimation = EstimateElementSize(params) * params.blockSize; + estimation = EstimateBlockSize(params, params.blockSize, n); actual = index_sq->getAllocationSize() - actual; // get the delta ASSERT_GT(actual, 0); ASSERT_GE(estimation * 1.01, actual); diff --git a/tests/unit/test_svs_multi.cpp b/tests/unit/test_svs_multi.cpp index 9c75ab8c3..bca0ee705 100644 --- a/tests/unit/test_svs_multi.cpp +++ b/tests/unit/test_svs_multi.cpp @@ -23,6 +23,14 @@ } \ } + +static size_t EstimateBlockSize(const SVSParams ¶ms, size_t block_size, size_t num_elements) { + const size_t reverse_edges_per_slot = SVSGraphBuilder::reverse_edges_element_size(); + const size_t reverse_edges_slots = svs::lib::SegmentedVector(num_elements).capacity(); + return (EstimateElementSize(params) - reverse_edges_per_slot) * block_size + + reverse_edges_per_slot * reverse_edges_slots; +} + // Log callback function to print non-debug log messages static void svsTestLogCallBackNoDebug(void *ctx, const char *level, const char *message) { if (level == nullptr || message == nullptr) { @@ -748,7 +756,7 @@ TYPED_TEST(SVSMultiTest, testSizeEstimation) { size_t actual = index->getAllocationSize(); ASSERT_EQ(estimation, actual); - estimation = EstimateElementSize(params) * bs; + estimation = EstimateBlockSize(params, bs, 1); GenerateAndAddVector(index, dim, 0); actual = index->getAllocationSize() - actual; // get the delta diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index ebf1d5497..d424d8124 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -28,6 +28,11 @@ } \ } +static size_t EstimateReverseEdgesSize(size_t num_elements) { + return SVSGraphBuilder::reverse_edges_element_size() * + svs::lib::SegmentedVector(num_elements).capacity(); +} + // Get available number of CPUs // Returns the number of logical processors on the process // Returns std::thread::hardware_concurrency() if the number of logical processors is not available @@ -1706,8 +1711,9 @@ TYPED_TEST(SVSTieredIndexTest, testSizeEstimation) { } mock_thread_pool.thread_pool_wait(); - // Estimate memory delta for filling up the first block and adding another block. - size_t estimation = VecSimIndex_EstimateElementSize(¶ms) * bs; + const size_t reverse_edges_per_slot = SVSGraphBuilder::reverse_edges_element_size(); + size_t estimation = (VecSimIndex_EstimateElementSize(¶ms) - reverse_edges_per_slot) * bs + + EstimateReverseEdgesSize(n + 1) - EstimateReverseEdgesSize(n); size_t before = index->getAllocationSize(); GenerateAndAddVector(index, dim, bs + n, bs + n); From cce885c615c81306c20139ad526865474e0b36b7 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Tue, 8 Sep 2026 02:05:46 -0700 Subject: [PATCH 24/64] fix ub --- src/VecSim/algorithms/svs/svs_tiered.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index f6e10ea30..447085c38 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -790,10 +790,7 @@ class TieredSVSIndex : public VecSimTieredIndex { memcpy(blob_copy.get(), this->frontendIndex->getDataByInternalId(job->id), data_size); this->flatIndexGuard.unlock_shared(); - { - std::shared_lock lock(updateJobMutex); - svs_index->addVector(blob_copy.get(), job->label); - } + svs_index->addVector(blob_copy.get(), job->label); // Remove the vector and the insert job from the flat buffer. this->flatIndexGuard.lock(); From f13bbb5f6130f9d433ad94a30774059dbd44fba4 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Tue, 8 Sep 2026 22:18:18 -0700 Subject: [PATCH 25/64] use SVS rc --- cmake/svs.cmake | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index 6174d827c..f6aebd87a 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -64,7 +64,7 @@ if(USE_SVS) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.31" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "21.0") if(GLIBCXX_VERSION VERSION_GREATER_EQUAL "11") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.3.2/svs-shared-library-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc1-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") else() message(STATUS "libstdc++ >= GCC 11 is required for Clang SVS binaries - disabling SVS_SHARED_LIB") set(SVS_SHARED_LIB OFF) @@ -76,12 +76,10 @@ if(USE_SVS) else() if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.28") if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "14.0") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.3.2/svs-shared-library-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc1-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") else() - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.3.2/svs-shared-library-reduced.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc1-reduced.tar.gz" CACHE STRING "SVS URL") endif() - elseif(GLIBC_VERSION VERSION_GREATER_EQUAL "2.26") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.3.0/svs-shared-library-0.3.0-reduced-glibc2_26.tar.gz" CACHE STRING "SVS URL") else() message(STATUS "GLIBC >= 2.26 is required for GCC SVS binaries - disabling SVS_SHARED_LIB") set(SVS_SHARED_LIB OFF) From eea54c56971d108f5b4e5d8bce6f22eee2c3913b Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 10 Sep 2026 09:31:54 +0200 Subject: [PATCH 26/64] fix TestIndexMultiValueJsonReload Co-authored-by: ethanglaser <42726565+ethanglaser@users.noreply.github.com> --- src/VecSim/algorithms/svs/svs_tiered.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 447085c38..d6388a721 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -957,7 +957,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // initSVSIndexWrapper() is called. { std::lock_guard lock(this->flatIndexGuard); - if (this->frontendIndex->isLabelExists(label)) { + if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { deleteAndUpdateInitIds(label); } ids_to_init_.insert(this->frontendIndex->indexSize()); From 7376b1f1942ef54cf60a231792ca4253399da1ff Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 10 Sep 2026 00:51:16 -0700 Subject: [PATCH 27/64] clang format --- src/VecSim/algorithms/svs/svs.h | 30 +++---- src/VecSim/algorithms/svs/svs_tiered.h | 103 +++++++++++++------------ src/VecSim/algorithms/svs/svs_utils.h | 6 +- tests/unit/test_svs.cpp | 10 ++- tests/unit/test_svs_multi.cpp | 1 - tests/unit/test_svs_tiered.cpp | 3 +- 6 files changed, 79 insertions(+), 74 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 2a367bad0..c84751cf5 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -41,7 +41,7 @@ struct SVSIndexBase virtual int addVectors(const void *vectors_data, const labelType *labels, size_t n) = 0; virtual int deleteVector(labelType label) = 0; virtual int deleteVectors(const labelType *labels, size_t n) = 0; - virtual void consolidate(const std::vector& labels) = 0; + virtual void consolidate(const std::vector &labels) = 0; virtual bool isLabelExists(labelType label) const = 0; virtual size_t indexStorageSize() const = 0; virtual size_t getParallelism() const = 0; @@ -126,13 +126,9 @@ class SVSIndex : public VecSimIndexAbstract, fl std::atomic impl_ready_{false}; - void setReady() { - this->impl_ready_.store(true, std::memory_order_release); - } + void setReady() { this->impl_ready_.store(true, std::memory_order_release); } - void setUnready() { - this->impl_ready_.store(false, std::memory_order_release); - } + void setUnready() { this->impl_ready_.store(false, std::memory_order_release); } static double toVecSimDistance(float v) { return svs_details::toVecSimDistance(v); } @@ -271,7 +267,8 @@ class SVSIndex : public VecSimIndexAbstract, fl { std::lock_guard lock(this->pimplGuard_); this->impl_ = std::move(svs_handler->impl); - if (this->impl_) setReady(); + if (this->impl_) + setReady(); } } @@ -323,7 +320,7 @@ class SVSIndex : public VecSimIndexAbstract, fl return n - deleted_num; } - void consolidate(const std::vector& labels) override { + void consolidate(const std::vector &labels) override { std::shared_lock lock(this->pimplGuard_); if (!ready()) return; @@ -421,9 +418,7 @@ class SVSIndex : public VecSimIndexAbstract, fl ~SVSIndex() = default; - bool ready() const override { - return this->impl_ready_.load(std::memory_order_acquire); - } + bool ready() const override { return this->impl_ready_.load(std::memory_order_acquire); } size_t indexSize() const override { return indexStorageSize(); } @@ -670,7 +665,6 @@ class SVSIndex : public VecSimIndexAbstract, fl } } - auto query_datum = std::span{static_cast(vector_data), this->dim}; { std::shared_lock lock(this->pimplGuard_); @@ -743,7 +737,7 @@ class SVSIndex : public VecSimIndexAbstract, fl auto processed_query_ptr = this->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); std::span query{static_cast(processed_query), - this->dim}; + this->dim}; // Base search parameters for the SVS iterator schedule. auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams, @@ -763,8 +757,8 @@ class SVSIndex : public VecSimIndexAbstract, fl // range search using epsilon const auto epsilon = queryParams && queryParams->svsRuntimeParams.epsilon != 0 - ? queryParams->svsRuntimeParams.epsilon - : this->epsilon; + ? queryParams->svsRuntimeParams.epsilon + : this->epsilon; const auto range_search_boundaries = radius * (1.0 + std::abs(epsilon)); bool keep_searching = true; @@ -780,8 +774,8 @@ class SVSIndex : public VecSimIndexAbstract, fl keep_searching = false; } } - // If search radius + epsilon is not exceeded, request SVS BatchIterator for the next - // batch + // If search radius + epsilon is not exceeded, request SVS BatchIterator for the + // next batch if (keep_searching) { svs_it.next(batch_size, cancel); if (cancel()) { diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index d6388a721..fd8b17239 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -21,7 +21,7 @@ struct SVSInsertJob : public AsyncJob { idType id; SVSInsertJob(std::shared_ptr allocator, labelType label_, idType id_, - JobCallback insertCb, VecSimIndex *index_) + JobCallback insertCb, VecSimIndex *index_) : AsyncJob(allocator, SVS_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} }; @@ -31,8 +31,9 @@ struct SVSInsertJob : public AsyncJob { struct SVSConsolidateJob : public AsyncJob { std::vector labels; - SVSConsolidateJob(std::shared_ptr allocator, const std::vector& labels_, - JobCallback insertCb, VecSimIndex *index_) + SVSConsolidateJob(std::shared_ptr allocator, + const std::vector &labels_, JobCallback insertCb, + VecSimIndex *index_) : AsyncJob(allocator, SVS_CONSOLIDATE_JOB, insertCb, index_), labels(labels_) {} }; @@ -40,12 +41,10 @@ struct SVSConsolidateJob : public AsyncJob { * Definition of a job that launches GC on SVS Index. */ struct SVSGCJob : public AsyncJob { - SVSGCJob(std::shared_ptr allocator, - JobCallback insertCb, VecSimIndex *index_) - : AsyncJob(allocator, SVS_GC_JOB, insertCb, index_) {} + SVSGCJob(std::shared_ptr allocator, JobCallback insertCb, VecSimIndex *index_) + : AsyncJob(allocator, SVS_GC_JOB, insertCb, index_) {} }; - /** * @class SVSMultiThreadJob * @brief Represents a multi-threaded asynchronous job for the SVS algorithm. @@ -422,8 +421,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::move(allocator)), index(index), flat_results(this->allocator), svs_results(this->allocator), flat_iterator(index->frontendIndex->newBatchIterator(query_vector, queryParams)), - svs_iterator(nullptr), - returned_results_set(this->allocator) { + svs_iterator(nullptr), returned_results_set(this->allocator) { if (queryParams) { this->queryParams = (VecSimQueryParams *)this->allocator->allocate(sizeof(VecSimQueryParams)); @@ -771,7 +769,8 @@ class TieredSVSIndex : public VecSimTieredIndex { auto svs_index = GetSVSIndex(); assert(svs_index->ready()); - // Note that accessing the job fields should occur with flat index guard held (here and later). + // Note that accessing the job fields should occur with flat index guard held (here and + // later). this->flatIndexGuard.lock_shared(); if (!job->isValid) { this->flatIndexGuard.unlock_shared(); @@ -817,12 +816,14 @@ class TieredSVSIndex : public VecSimTieredIndex { this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); int deleted = this->frontendIndex->deleteVectorById(job->label, job->id); if (deleted && job->id != this->frontendIndex->indexSize()) { - // If the vector removal caused a swap with the last id, update the relevant insert job. + // If the vector removal caused a swap with the last id, update the relevant insert + // job. this->updateInsertJobInternalId(this->frontendIndex->indexSize(), job->id, last_vec_label); } } else { - // Remove the current job from the invalid jobs' lookup, as we are about to delete it now. + // Remove the current job from the invalid jobs' lookup, as we are about to delete it + // now. this->invalidJobsLookupGuard.lock(); this->invalidJobs.erase(job->id); this->invalidJobsLookupGuard.unlock(); @@ -860,7 +861,8 @@ class TieredSVSIndex : public VecSimTieredIndex { if (!labels_to_move.empty()) { auto svs_index = GetSVSIndex(); svs_index->setParallelism(std::min(availableThreads, labels_to_move.size())); - assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); + assert(labels_to_move.size() == + vectors_to_move.size() / this->frontendIndex->getDim()); // The backend may already have been initialized while this job was queued if (svs_index->ready()) { svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), @@ -885,7 +887,8 @@ class TieredSVSIndex : public VecSimTieredIndex { this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); int deleted = this->frontendIndex->deleteVectorById(label, id); if (deleted && id != this->frontendIndex->indexSize()) { - // If the vector removal caused a swap with the last id, update the relevant insert job. + // If the vector removal caused a swap with the last id, update the relevant + // insert job. this->updateInsertJobInternalId(this->frontendIndex->indexSize(), id, last_vec_label); } @@ -893,8 +896,9 @@ class TieredSVSIndex : public VecSimTieredIndex { } } - assert(total_deleted == std::count_if(labels_to_move.begin(), labels_to_move.end(), - [](labelType label) { return label != SKIP_LABEL; }) && + assert(total_deleted == + std::count_if(labels_to_move.begin(), labels_to_move.end(), + [](labelType label) { return label != SKIP_LABEL; }) && "Deleted vectors count does not match the number of labels to delete"); } // release frontend index executeTracingCallback("UpdateJob::after_add_to_svs"); @@ -905,16 +909,14 @@ class TieredSVSIndex : public VecSimTieredIndex { const TieredIndexParams &tiered_index_params, std::shared_ptr allocator) : Base(svs_index, bf_index, tiered_index_params, allocator), - uncompletedJobs(this->allocator), - labelToInsertJobs(this->allocator), - invalidJobs(this->allocator), - currInvalidJobId(0) { + uncompletedJobs(this->allocator), labelToInsertJobs(this->allocator), + invalidJobs(this->allocator), currInvalidJobId(0) { const auto &tiered_svs_params = tiered_index_params.specificParams.tieredSVSParams; // If flatBufferLimit is not initialized (0), use the default update threshold. flat_buffer_bound = tiered_index_params.flatBufferLimit == 0 - ? SVS_VAMANA_DEFAULT_UPDATE_THRESHOLD - : tiered_index_params.flatBufferLimit; + ? SVS_VAMANA_DEFAULT_UPDATE_THRESHOLD + : tiered_index_params.flatBufferLimit; this->updateTriggerThreshold = tiered_svs_params.updateTriggerThreshold == 0 @@ -951,13 +953,15 @@ class TieredSVSIndex : public VecSimTieredIndex { // so make the minimal needed lock here. // Backend index initialization data have to be buffered for proper // compression/training. - if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { + if ((!svs_index->ready()) && + (!this->backendInitSubmited.load(std::memory_order_acquire))) { // If backend index size is 0, first collect vectors in frontend index // lock in scope to ensure that these will be released before // initSVSIndexWrapper() is called. { std::lock_guard lock(this->flatIndexGuard); - if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { + if (!this->frontendIndex->isMultiValue() && + this->frontendIndex->isLabelExists(label)) { deleteAndUpdateInitIds(label); } ids_to_init_.insert(this->frontendIndex->indexSize()); @@ -998,8 +1002,10 @@ class TieredSVSIndex : public VecSimTieredIndex { if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { // Add vector to the frontend index. std::lock_guard lock(this->flatIndexGuard); - if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { - if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { + if ((!svs_index->ready()) && + (!this->backendInitSubmited.load(std::memory_order_acquire))) { + if (!this->frontendIndex->isMultiValue() && + this->frontendIndex->isLabelExists(label)) { deleteAndUpdateInitIds(label); } ids_to_init_.insert(this->frontendIndex->indexSize()); @@ -1041,20 +1047,21 @@ class TieredSVSIndex : public VecSimTieredIndex { return ret; } - // Overwrite the vector and invalidate its only pending job (since we are not in MULTI). - // Label exists, but job doesn't. It means the label is used for initialization. - // Just create new job, this job will remove duplicate from the backend. + // Overwrite the vector and invalidate its only pending job (since we are not in + // MULTI). Label exists, but job doesn't. It means the label is used for + // initialization. Just create new job, this job will remove duplicate from the + // backend. auto *old_job = this->labelToInsertJobs.at(label).at(0); old_job->id = this->setAndSaveInvalidJob(old_job); this->labelToInsertJobs.erase(label); ret = 0; - // We are going to update the internal id that currently holds the vector associated with - // the given label. + // We are going to update the internal id that currently holds the vector associated + // with the given label. new_flat_id = dynamic_cast *>(this->frontendIndex) ->getIdOfLabel(label); - // If we are adding a new element (rather than updating an exiting one) we may need to - // increase index capacity. + // If we are adding a new element (rather than updating an exiting one) we may need + // to increase index capacity. } // If this label already exists, this will do overwrite. ret += this->frontendIndex->addVector(blob, label); @@ -1062,10 +1069,11 @@ class TieredSVSIndex : public VecSimTieredIndex { AsyncJob *new_insert_job = new (this->allocator) SVSInsertJob(this->allocator, label, new_flat_id, executeInsertJobWrapper, this); - // Save a pointer to the job, so that if the vector is overwritten, we'll have an indication. + // Save a pointer to the job, so that if the vector is overwritten, we'll have an + // indication. if (this->labelToInsertJobs.find(label) != this->labelToInsertJobs.end()) { - // There's already a pending insert job for this label, add another one (without overwrite, - // only possible in multi index) + // There's already a pending insert job for this label, add another one (without + // overwrite, only possible in multi index) assert(this->backendIndex->isMultiValue()); this->labelToInsertJobs.at(label).push_back((SVSInsertJob *)new_insert_job); } else { @@ -1080,8 +1088,8 @@ class TieredSVSIndex : public VecSimTieredIndex { // we submit the insert job. if (!this->backendIndex->isMultiValue()) { if (svs_index->ready()) { - // If we removed the previous vector from both svs and flat in the overwrite process, - // we still return 0 (not -1). + // If we removed the previous vector from both svs and flat in the overwrite + // process, we still return 0 (not -1). auto deleted = svs_index->deleteVector(label); if (deleted > 0) scheduleSVSIndexConsolidate(label); @@ -1137,14 +1145,17 @@ class TieredSVSIndex : public VecSimTieredIndex { if (this->frontendIndex->isLabelExists(label)) { this->flatIndexGuard.unlock_shared(); std::lock_guard flat_lock{this->flatIndexGuard}; - // Check again if the label exists, as it may have been removed while we released the lock. + // Check again if the label exists, as it may have been removed while we released the + // lock. if (this->frontendIndex->isLabelExists(label)) { auto deleting_ids = this->frontendIndex->getElementIds(label); - if (deleting_ids.size() == 0) return 0; + if (deleting_ids.size() == 0) + return 0; // assert if all elements of deleting_ids are unique - assert(std::set(deleting_ids.begin(), deleting_ids.end()).size() == deleting_ids.size() && - "deleting_ids should contain unique ids"); + assert(std::set(deleting_ids.begin(), deleting_ids.end()).size() == + deleting_ids.size() && + "deleting_ids should contain unique ids"); // If id is deleted, don't use it for initialization for (idType id : deleting_ids) { @@ -1391,11 +1402,7 @@ class TieredSVSIndex : public VecSimTieredIndex { scheduleSVSIndexGC(); } - void acquireSharedLocks() override { - this->flatIndexGuard.lock_shared(); - } + void acquireSharedLocks() override { this->flatIndexGuard.lock_shared(); } - void releaseSharedLocks() override { - this->flatIndexGuard.unlock_shared(); - } + void releaseSharedLocks() override { this->flatIndexGuard.unlock_shared(); } }; diff --git a/src/VecSim/algorithms/svs/svs_utils.h b/src/VecSim/algorithms/svs/svs_utils.h index 45860c9ba..670ea172c 100644 --- a/src/VecSim/algorithms/svs/svs_utils.h +++ b/src/VecSim/algorithms/svs/svs_utils.h @@ -218,7 +218,7 @@ inline std::pair isSVSQuantBitsSupported(VecSimSvsQuan template svs::concurrent::SegmentedBlocked> make_segmented_blocked_allocator_handle(const svs::data::BlockingParameters ¶meters, - Alloc alloc) { + Alloc alloc) { using handle_type = svs::AllocatorHandle; return svs::concurrent::SegmentedBlocked{ parameters, svs::make_allocator_handle(std::move(alloc))}; @@ -338,8 +338,8 @@ struct SVSGraphBuilder { // based on the data types, which we found to perform better through heuristic analysis. auto prefetch_parameters = svs::index::vamana::extensions::estimate_prefetch_parameters(data); - auto builder = svs::concurrent::VamanaBuilder( - graph, data, std::move(distance), parameters, threadpool, prefetch_parameters, logger); + auto builder = svs::concurrent::VamanaBuilder(graph, data, std::move(distance), parameters, + threadpool, prefetch_parameters, logger); // Specific to the Vamana algorithm: // It builds in two rounds, one with alpha=1 and the second time with the user/config diff --git a/tests/unit/test_svs.cpp b/tests/unit/test_svs.cpp index 3105372b2..953a95cc4 100644 --- a/tests/unit/test_svs.cpp +++ b/tests/unit/test_svs.cpp @@ -1583,7 +1583,8 @@ TYPED_TEST(SVSTest, test_svs_parameter_combinations_and_defaults) { // Test: All default parameters (zeros/unset) {"all_defaults", { - .dim = dim, .metric = VecSimMetric_L2, + .dim = dim, + .metric = VecSimMetric_L2, // All other parameters left as default (0/unset) }, {.quantBits = get<0>(svs_details::isSVSQuantBitsSupported(TypeParam::get_quant_bits())), @@ -1768,7 +1769,8 @@ TYPED_TEST(SVSTest, test_svs_parameter_consistency_across_metrics) { // Create index with default parameters for this metric SVSParams params = { - .dim = dim, .metric = metric, + .dim = dim, + .metric = metric, // All other parameters use defaults }; @@ -3369,7 +3371,9 @@ TEST(SVSTest, NumThreadsParamIgnored) { // Verify: creating an index without setting num_threads produces no warning. captured_log.clear(); SVSParams svs_params_default = { - .type = VecSimType_FLOAT32, .dim = 4, .metric = VecSimMetric_L2, + .type = VecSimType_FLOAT32, + .dim = 4, + .metric = VecSimMetric_L2, // num_threads left as 0 (default / unset) }; VecSimParams params_default{.algo = VecSimAlgo_SVS, diff --git a/tests/unit/test_svs_multi.cpp b/tests/unit/test_svs_multi.cpp index bca0ee705..0ed155ba0 100644 --- a/tests/unit/test_svs_multi.cpp +++ b/tests/unit/test_svs_multi.cpp @@ -23,7 +23,6 @@ } \ } - static size_t EstimateBlockSize(const SVSParams ¶ms, size_t block_size, size_t num_elements) { const size_t reverse_edges_per_slot = SVSGraphBuilder::reverse_edges_element_size(); const size_t reverse_edges_slots = svs::lib::SegmentedVector(num_elements).capacity(); diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index d424d8124..abea602f4 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -1282,7 +1282,8 @@ TYPED_TEST(SVSTieredIndexTestBasic, markedDeleted) { ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 0); // Move vectors to the backend - while (mock_thread_pool.jobQ.size() > 0) mock_thread_pool.thread_iteration(); + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), n); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->indexSize(), n); From a46de7f195ae99f71af9ee13441ab12d3e1f1225 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 10 Sep 2026 01:08:17 -0700 Subject: [PATCH 28/64] cland-format-18 --- tests/unit/test_svs.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_svs.cpp b/tests/unit/test_svs.cpp index 953a95cc4..3105372b2 100644 --- a/tests/unit/test_svs.cpp +++ b/tests/unit/test_svs.cpp @@ -1583,8 +1583,7 @@ TYPED_TEST(SVSTest, test_svs_parameter_combinations_and_defaults) { // Test: All default parameters (zeros/unset) {"all_defaults", { - .dim = dim, - .metric = VecSimMetric_L2, + .dim = dim, .metric = VecSimMetric_L2, // All other parameters left as default (0/unset) }, {.quantBits = get<0>(svs_details::isSVSQuantBitsSupported(TypeParam::get_quant_bits())), @@ -1769,8 +1768,7 @@ TYPED_TEST(SVSTest, test_svs_parameter_consistency_across_metrics) { // Create index with default parameters for this metric SVSParams params = { - .dim = dim, - .metric = metric, + .dim = dim, .metric = metric, // All other parameters use defaults }; @@ -3371,9 +3369,7 @@ TEST(SVSTest, NumThreadsParamIgnored) { // Verify: creating an index without setting num_threads produces no warning. captured_log.clear(); SVSParams svs_params_default = { - .type = VecSimType_FLOAT32, - .dim = 4, - .metric = VecSimMetric_L2, + .type = VecSimType_FLOAT32, .dim = 4, .metric = VecSimMetric_L2, // num_threads left as 0 (default / unset) }; VecSimParams params_default{.algo = VecSimAlgo_SVS, From ffb1e845263df3cf952283797bfc75fd03f742ef Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Thu, 10 Sep 2026 06:09:35 -0700 Subject: [PATCH 29/64] num_marked_deleted tracks not compacted entries now --- src/VecSim/algorithms/svs/svs.h | 3 +-- tests/unit/test_svs_tiered.cpp | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index c84751cf5..b7c19a1ed 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -325,8 +325,7 @@ class SVSIndex : public VecSimIndexAbstract, fl if (!ready()) return; - size_t n_consolidated = impl_->consolidate(labels); - num_marked_deleted.fetch_sub(n_consolidated, std::memory_order_relaxed); + impl_->consolidate(labels); } int deleteVectorImpl(const labelType label) { diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index abea602f4..63c69efe1 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -3096,6 +3096,7 @@ TYPED_TEST(SVSTieredIndexTest, writeInPlaceMode) { if (!TypeParam::isMulti()) { TEST_DATA_T overwritten_vec[] = {1, 1, 1, 1}; tiered_index->addVector(overwritten_vec, vec_label); + expected_marked_deleted++; ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 3); ASSERT_EQ(tiered_index->indexSize(), 3); ASSERT_EQ(tiered_index->indexLabelCount(), 2); @@ -3106,6 +3107,7 @@ TYPED_TEST(SVSTieredIndexTest, writeInPlaceMode) { // Validate that the vector is deleted and consolidated synchronously (in-place mode), // so no marked-deleted entry remains. tiered_index->deleteVector(vec_label); + expected_marked_deleted++; ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), TypeParam::isMulti() ? 2 : 3); ASSERT_EQ(tiered_index->indexLabelCount(), 1); From 5bf39ab8e0ca03271a45615ec99430020e5be4a0 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Fri, 11 Sep 2026 05:06:49 -0700 Subject: [PATCH 30/64] clean-up and fix --- src/VecSim/algorithms/svs/svs.h | 29 ++---- src/VecSim/algorithms/svs/svs_tiered.h | 139 ++++++++----------------- src/VecSim/algorithms/svs/svs_utils.h | 1 - tests/unit/test_svs_tiered.cpp | 2 +- 4 files changed, 52 insertions(+), 119 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index b7c19a1ed..00afacc95 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -93,8 +93,6 @@ class SVSIndex : public VecSimIndexAbstract, fl using graph_builder_t = SVSGraphBuilder; using graph_type = typename graph_builder_t::graph_type; - // The concurrent index mirrors the upstream template signature and member surface, so - // the only change needed here is the namespace. using impl_type = std::conditional_t< isMulti, svs::concurrent::MultiMutableVamanaIndex, @@ -321,6 +319,8 @@ class SVSIndex : public VecSimIndexAbstract, fl } void consolidate(const std::vector &labels) override { + // There is documentation for consolidate(): + // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.consolidate std::shared_lock lock(this->pimplGuard_); if (!ready()) return; @@ -369,10 +369,9 @@ class SVSIndex : public VecSimIndexAbstract, fl if (indexLabelCount() == 0) { std::lock_guard lock(this->pimplGuard_); if (indexLabelCountUnsafe() == 0) { - { - setUnready(); - this->impl_.reset(); - } + setUnready(); + this->impl_.reset(); + num_marked_deleted.store(0, std::memory_order_relaxed); return; } @@ -657,19 +656,14 @@ class SVSIndex : public VecSimIndexAbstract, fl if (!ready()) { return std::numeric_limits::quiet_NaN(); } - { - std::shared_lock lock(this->pimplGuard_); - if (!impl_->has_id(label)) { - return std::numeric_limits::quiet_NaN(); - } + std::shared_lock lock(this->pimplGuard_); + if (!impl_->has_id(label)) { + return std::numeric_limits::quiet_NaN(); } auto query_datum = std::span{static_cast(vector_data), this->dim}; - { - std::shared_lock lock(this->pimplGuard_); - auto dist = impl_->get_distance(label, query_datum); - return toVecSimDistance(dist); - } + auto dist = impl_->get_distance(label, query_datum); + return toVecSimDistance(dist); } VecSimQueryReply *topKQuery(const void *queryBlob, size_t k, @@ -840,9 +834,6 @@ class SVSIndex : public VecSimIndexAbstract, fl void runGC() override { if (ready()) { std::shared_lock lock(this->pimplGuard_); - // There is documentation for consolidate(): - // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.consolidate - // impl_->consolidate(); // There is documentation for compact(): // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.compact impl_->compact(); diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index fd8b17239..80dcae8ea 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -11,6 +11,7 @@ #include #include #include +#include #include /** @@ -19,6 +20,7 @@ struct SVSInsertJob : public AsyncJob { labelType label; idType id; + std::atomic executing{false}; SVSInsertJob(std::shared_ptr allocator, labelType label_, idType id_, JobCallback insertCb, VecSimIndex *index_) @@ -37,14 +39,6 @@ struct SVSConsolidateJob : public AsyncJob { : AsyncJob(allocator, SVS_CONSOLIDATE_JOB, insertCb, index_), labels(labels_) {} }; -/** - * Definition of a job that launches GC on SVS Index. - */ -struct SVSGCJob : public AsyncJob { - SVSGCJob(std::shared_ptr allocator, JobCallback insertCb, VecSimIndex *index_) - : AsyncJob(allocator, SVS_GC_JOB, insertCb, index_) {} -}; - /** * @class SVSMultiThreadJob * @brief Represents a multi-threaded asynchronous job for the SVS algorithm. @@ -258,23 +252,6 @@ class TieredSVSIndex : public VecSimTieredIndex { using backend_index_t = VecSimIndexAbstract; using svs_index_t = SVSIndexBase; - // swaps_journal is used by updateSVSIndex() to track vectors swap operations that were done in - // the Flat index during SVS index updating. - // The journal contains tuples of (label, oldId, newId). - // oldId is the index of the label in flat index before the swap. - // newId is the index of the label in flat index after the swap. - // if oldId == newId, it means that the vector was not moved in the Flat index, but was removed - // from the end of the Flat index (no id swaps occurred internally). - // if label == SKIP_LABEL, it means that the vector was not moved in the Flat index, but - // updated in-place (hence was already removed and no need to remove it again) - using swap_record = std::tuple; - constexpr static size_t SKIP_LABEL = std::numeric_limits::max(); - std::vector swaps_journal; - // deleted_labels_journal is used by initSVSIndex() to track vectors that were deleted from - // Flat index during SVS index updating. The journal contains the deleted labels. These labels - // are used to delete the same vectors from the SVS index at the end of the update. - std::vector deleted_labels_journal; - size_t trainingTriggerThreshold; size_t updateTriggerThreshold; size_t updateJobWaitTime; @@ -726,37 +703,9 @@ class TieredSVSIndex : public VecSimTieredIndex { } private: - static void applySwapsToLabelsArray(std::vector &labels, - const std::vector &swaps) { - // Enumerate journal and reflect swaps in the labels. - // The journal contains tuples of (label, oldId, newId). - // oldId is the index of the label in flat index before the swap. - // newId is the index of the label in flat index after the swap. - for (const auto &p : swaps) { - auto oldId = std::get<1>(p); - auto newId = std::get<2>(p); - - if (oldId == newId || oldId >= labels.size()) { - // If oldId == newId, it means that the vector was not moved in the Flat index, - // but was removed or updated in-place. - // If oldId is out of bounds - new vector was added and swapped meanwhile. - // In both cases, we should not touch flat index at the new position. - if (newId < labels.size()) { - labels[newId] = SKIP_LABEL; - } - continue; // Next swap record. - } - - // Real swap case. - // If oldId != newId and oldId is in bounds, it means that the tracked vector was moved - // So, move the label to new position and skip the old position from deletions. - // NOTE: labels[oldId] can be SKIP_LABEL, but we still need to move it to newId. - labels[newId] = labels[oldId]; - labels[oldId] = SKIP_LABEL; - } - } - - idType setAndSaveInvalidJob(AsyncJob *job) { + idType setAndSaveInvalidJob(SVSInsertJob *job) { + // wait until insertion finishes + while (job->executing.load(std::memory_order_acquire)) {std::this_thread::yield();} this->invalidJobsLookupGuard.lock(); job->isValid = false; idType curInvalidId = currInvalidJobId++; @@ -787,10 +736,13 @@ class TieredSVSIndex : public VecSimTieredIndex { size_t data_size = this->frontendIndex->getStoredDataSize(); auto blob_copy = this->getAllocator()->allocate_unique(data_size); memcpy(blob_copy.get(), this->frontendIndex->getDataByInternalId(job->id), data_size); + + job->executing.store(true, std::memory_order_release); this->flatIndexGuard.unlock_shared(); svs_index->addVector(blob_copy.get(), job->label); + job->executing.store(false, std::memory_order_release); // Remove the vector and the insert job from the flat buffer. this->flatIndexGuard.lock(); // The job might have been invalidated due to overwrite in the meantime. In this case, @@ -882,23 +834,19 @@ class TieredSVSIndex : public VecSimTieredIndex { idType id = *it; auto label = this->frontendIndex->getVectorLabel(id); // Delete the vector from the frontend index if not in-place updated. - if (label != SKIP_LABEL) { - labelType last_vec_label = - this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); - int deleted = this->frontendIndex->deleteVectorById(label, id); - if (deleted && id != this->frontendIndex->indexSize()) { - // If the vector removal caused a swap with the last id, update the relevant - // insert job. - this->updateInsertJobInternalId(this->frontendIndex->indexSize(), id, - last_vec_label); - } - total_deleted += deleted; + labelType last_vec_label = + this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); + int deleted = this->frontendIndex->deleteVectorById(label, id); + if (deleted && id != this->frontendIndex->indexSize()) { + // If the vector removal caused a swap with the last id, update the relevant + // insert job. + this->updateInsertJobInternalId(this->frontendIndex->indexSize(), id, + last_vec_label); } + total_deleted += deleted; } - assert(total_deleted == - std::count_if(labels_to_move.begin(), labels_to_move.end(), - [](labelType label) { return label != SKIP_LABEL; }) && + assert(total_deleted == labels_to_move.size() && "Deleted vectors count does not match the number of labels to delete"); } // release frontend index executeTracingCallback("UpdateJob::after_add_to_svs"); @@ -936,9 +884,20 @@ class TieredSVSIndex : public VecSimTieredIndex { this->updateJobWaitTime = tiered_svs_params.updateJobWaitTime == 0 ? SVS_DEFAULT_UPDATE_JOB_WAIT_TIME : tiered_svs_params.updateJobWaitTime; + } + + ~TieredSVSIndex() { + // Delete all the pending insert jobs. + for (auto &jobs : this->labelToInsertJobs) { + for (auto *job : jobs.second) { + delete job; + } + } - // Reserve space for the journal to avoid reallocation. - this->swaps_journal.reserve(this->trainingTriggerThreshold); + // Delete all the pending invalid jobs. + for (auto &it : this->invalidJobs) { + delete it.second; + } } int addVector(const void *blob, labelType label) override { @@ -948,14 +907,11 @@ class TieredSVSIndex : public VecSimTieredIndex { // In-Place mode - add vector syncronously to the backend index. if (this->getWriteMode() == VecSim_WriteInPlace) { - // It is ok to lock everything at once for in-place mode, - // but we will have to unlock averything before calling initSVSIndexWrapper() - // so make the minimal needed lock here. // Backend index initialization data have to be buffered for proper // compression/training. if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { - // If backend index size is 0, first collect vectors in frontend index + // First collect vectors in frontend index // lock in scope to ensure that these will be released before // initSVSIndexWrapper() is called. { @@ -1106,17 +1062,15 @@ class TieredSVSIndex : public VecSimTieredIndex { void deleteAndUpdateInitIds(labelType label) { auto deleting_ids = this->frontendIndex->getElementIds(label); - // Sort deleting_ids by id descending order - std::sort(deleting_ids.begin(), deleting_ids.end(), - [](const auto &a, const auto &b) { return a > b; }); + std::sort(deleting_ids.begin(), deleting_ids.end()); // Delete vector from the frontend index. auto updated_ids = this->frontendIndex->deleteVectorAndGetUpdatedIds(label); assert(std::all_of(updated_ids.begin(), updated_ids.end(), [&deleting_ids](const auto &pair) { - return std::find(deleting_ids.begin(), deleting_ids.end(), - pair.first) != deleting_ids.end(); + return std::binary_search(deleting_ids.begin(), deleting_ids.end(), + pair.first); }) && "updated_ids should be a subset of deleting_ids"); @@ -1233,24 +1187,13 @@ class TieredSVSIndex : public VecSimTieredIndex { .updateTriggerThreshold = this->updateTriggerThreshold, .updateJobWaitTime = this->updateJobWaitTime, }; - { - // Use try_lock to avoid blocking the main thread during long-running - // training operations. initSVSIndexWrapper holds updateJobMutex for - // the entire training duration (which can take 40-85s on slow machines). - // If the mutex is held, training is actively running, so we report - // indexUpdateScheduled = true (BACKGROUND_INDEXING = 1). - std::unique_lock lock(this->updateJobMutex, std::try_to_lock); - if (lock.owns_lock()) { - svsTieredInfo.indexUpdateScheduled = this->indexUpdateScheduled.test(); - } else { - // Mutex is held by initSVSIndexWrapper — training is in progress. - svsTieredInfo.indexUpdateScheduled = true; - } - } + + svsTieredInfo.indexUpdateScheduled = + (info.tieredInfo.frontendCommonInfo.indexSize > 0) && + this->backendInitSubmited.load(std::memory_order_acquire); info.tieredInfo.specificTieredBackendInfo.svsTieredInfo = svsTieredInfo; - // Background indexing is in progress whenever the flat buffer is non-empty: every vector - // buffered there has either a pending per-vector insert job or a batch-init job that will - // drain it into the backend. + + // Background indexing is in progress whenever the flat buffer is non-empty info.tieredInfo.backgroundIndexing = info.tieredInfo.frontendCommonInfo.indexSize > 0 ? VecSimBool_TRUE : VecSimBool_FALSE; return info; diff --git a/src/VecSim/algorithms/svs/svs_utils.h b/src/VecSim/algorithms/svs/svs_utils.h index 670ea172c..232240ae9 100644 --- a/src/VecSim/algorithms/svs/svs_utils.h +++ b/src/VecSim/algorithms/svs/svs_utils.h @@ -375,7 +375,6 @@ struct SVSGraphBuilder { } // Per-node size of the concurrent graph's reverse-edge index. - // static constexpr size_t reverse_edges_element_size() { using reverse_edges_type = typename graph_type::reverse_edges_type; return sizeof(typename reverse_edges_type::list_type) + sizeof(svs::concurrent::SpinLock); diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 63c69efe1..47d69004b 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -3303,7 +3303,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, runGCAPI) { auto jobs_before_gc = mock_thread_pool.jobQ.size(); // Run the GC API call, expect that we will clean up the SVS index. VecSimTieredIndex_GC(tiered_index); - // Expected that a single GC job was added to the queue. + // Expected that a multi-thread GC job was added to the queue. ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + mock_thread_pool.thread_pool_size); // Run GC twice. VecSimTieredIndex_GC(tiered_index); From b4a2d832aebe4c9d03f21fc6b18ce24d49d07977 Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Fri, 11 Sep 2026 11:16:17 -0700 Subject: [PATCH 31/64] update SVS submodule reference --- deps/ScalableVectorSearch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch index 7786d43b9..389597b59 160000 --- a/deps/ScalableVectorSearch +++ b/deps/ScalableVectorSearch @@ -1 +1 @@ -Subproject commit 7786d43b98ac9769ad7668d0d4896143cfb2f167 +Subproject commit 389597b59cbfe87c6887da262673ef386261947e From a544c4c5ef007be3c4f8112e6a5b2cd89c2f102c Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Fri, 11 Sep 2026 12:22:25 -0700 Subject: [PATCH 32/64] Fix semantic fallout from the main merge Three things the merge could not resolve textually, because both sides changed code the other side never saw. test_svs_tiered.cpp: #1033's new getDataByLabelReadsSvsBackendWhenUncompressed calls scheduleSVSIndexUpdate(), renamed to scheduleSVSIndexInit() on this branch. Git merged the call site cleanly and the build broke. A rename is enough -- CreateIndexInstance already uses the same add / schedule / single thread_iteration shape against the init-job model, so one iteration does complete the ingest. The other two tests #1033 added need no change: one never reaches the backend, and the compressed-split-label one's premises still hold here (threshold 1 latches backendInitSubmited on the first add, and the second add is buffered because the flat index is below its bound). svs.h: #1033's appendStoredDataByLabel guards impl_ with a bare null check, which was sufficient on main -- main has no pimplGuard_. Here every other const reader of impl_ takes it shared, and markIndexUpdate takes it exclusively to reset impl_ once the last label is deleted. The spans append_datum copies from point into the dataset, so a getDataByLabel racing that delete walked a destroyed index; mainIndexGuard in the tiered caller does not cover this, since backend writes on this branch deliberately do not take it. Now takes pimplGuard_ shared, testing impl_ under the lock rather than before it -- neither caller holds the guard, so there is no recursive acquisition. Also updates a comment naming updateSVSIndexWrapper. Note for follow-up, not fixed here: the pre-existing readers (isLabelExists, getDistanceFrom_Unsafe) test ready() *outside* pimplGuard_ and dereference impl_ after acquiring it, so a reset landing in that window still null-derefs. That predates the merge and belongs with the other lock-free review items. Co-Authored-By: Claude Opus 5 --- src/VecSim/algorithms/svs/svs.h | 5 +++++ tests/unit/test_svs_tiered.cpp | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 574db33da..d52ec7be6 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -849,6 +849,11 @@ class SVSIndex : public VecSimIndexAbstract, fl template void appendStoredDataByLabel(labelType label, std::vector> &vectors_output) const { + // The spans handed to `append_datum` point straight into the dataset, so `impl_` has to + // stay alive for the whole walk: `markIndexUpdate` takes `pimplGuard_` exclusively to + // reset it once the last label is gone. Test it under the lock rather than before it -- a + // check outside the lock can pass and then block until after the reset. + std::shared_lock lock(this->pimplGuard_); if (!impl_) { return; } diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 81c5bdb0c..4b817d472 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -407,7 +407,7 @@ TYPED_TEST(SVSTieredIndexTest, TestDebugInfoThreadCountWriteInPlace) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); } // Verify: numThreads (pool size) unchanged, lastReservedThreads (parallelism) reflects - // we only used one thread (write-in-place calls updateSVSIndexWrapper with availableThreads=1). + // we only used one thread (write-in-place calls initSVSIndexWrapper with availableThreads=1). backendIndexInfo = tiered_index->GetBackendIndex()->debugInfo(); ASSERT_EQ(backendIndexInfo.svsInfo.numThreads, num_threads); ASSERT_EQ(backendIndexInfo.svsInfo.lastReservedThreads, 1); @@ -549,7 +549,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, getDataByLabelReadsSvsBackendWhenUncompresse ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 1); // Ingest it into the SVS backend. - tiered_index->scheduleSVSIndexUpdate(); + tiered_index->scheduleSVSIndexInit(); mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 1); From 3e0a2cfc4e86037297cd4404959ed548dc18c11f Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Fri, 11 Sep 2026 12:22:46 -0700 Subject: [PATCH 33/64] Satisfy check-format The spin-wait added in "clean-up and fix" kept its body on the same line as the `while`. Only violation in the tree under clang-format-18, the version CI installs via llvm.sh on ubuntu-24.04. Co-Authored-By: Claude Opus 5 --- src/VecSim/algorithms/svs/svs_tiered.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 80dcae8ea..1e1783cca 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -705,7 +705,9 @@ class TieredSVSIndex : public VecSimTieredIndex { private: idType setAndSaveInvalidJob(SVSInsertJob *job) { // wait until insertion finishes - while (job->executing.load(std::memory_order_acquire)) {std::this_thread::yield();} + while (job->executing.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } this->invalidJobsLookupGuard.lock(); job->isValid = false; idType curInvalidId = currInvalidJobId++; From fb1f99eaca3961408ba2bda76f8ae059b1487e37 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 14 Sep 2026 00:20:22 -0700 Subject: [PATCH 34/64] improve svs pimpl guarding for concurent execution --- src/VecSim/algorithms/svs/svs.h | 264 ++++++++---------- .../algorithms/svs/svs_batch_iterator.h | 13 +- .../algorithms/svs/svs_serializer_impl.h | 33 ++- 3 files changed, 148 insertions(+), 162 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 00afacc95..a9b608697 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include "svs/index/vamana/dynamic_index.h" @@ -118,15 +120,21 @@ class SVSIndex : public VecSimIndexAbstract, fl // SVS thread pool VecSimSVSThreadPool threadpool_; svs::logging::logger_ptr logger_; - // SVS Index implementation instance - std::unique_ptr impl_; + // SVS Index implementation instance. + // initImpl() needs at least one point to compute an entry point. + // so impl_ is null until the first insert + std::shared_ptr impl_; + mutable std::shared_mutex pimplGuard_; + // Serialize mutators of the impl_ pointer (exclusive) + // vs add_points() (shared) to avoid silent drops of additions + // Lock order: implMutationGuard_ -> pimplGuard_. + std::shared_mutex implMutationGuard_; - std::atomic impl_ready_{false}; - - void setReady() { this->impl_ready_.store(true, std::memory_order_release); } - - void setUnready() { this->impl_ready_.store(false, std::memory_order_release); } + std::shared_ptr getImpl() const { + std::shared_lock lock(this->pimplGuard_); + return this->impl_; + } static double toVecSimDistance(float v) { return svs_details::toVecSimDistance(v); } @@ -169,7 +177,7 @@ class SVSIndex : public VecSimIndexAbstract, fl // Create SVS index instance with initial data // Data should not be empty template - std::unique_ptr initImpl(const Dataset &points, + std::shared_ptr initImpl(const Dataset &points, std::span ids) const { svs::threads::ThreadPoolHandle threadpool_handle{VecSimSVSThreadPool{threadpool_}}; @@ -190,7 +198,7 @@ class SVSIndex : public VecSimIndexAbstract, fl this->blockSize, this->getAllocator(), logger_); // Create SVS MutableIndex instance - auto impl = std::make_unique(std::move(graph), std::move(data), entry_point, + auto impl = std::make_shared(std::move(graph), std::move(data), entry_point, std::move(distance), ids, threadpool_, logger_); // Set SVS MutableIndex build parameters to be used in future updates @@ -233,8 +241,8 @@ class SVSIndex : public VecSimIndexAbstract, fl // Handler to manage SVS implementation instance struct SVSImplHandler : public SVSIndexBase::ImplHandler { - std::unique_ptr impl; - SVSImplHandler(std::unique_ptr impl) : impl{std::move(impl)} {} + std::shared_ptr impl; + SVSImplHandler(std::shared_ptr impl) : impl{std::move(impl)} {} }; std::unique_ptr createImpl(const void *vectors_data, const labelType *labels, @@ -254,20 +262,18 @@ class SVSIndex : public VecSimIndexAbstract, fl } void setImpl(std::unique_ptr handler) override { - if (ready()) { - throw std::logic_error("SVSIndex::setImpl called on non-empty impl_"); - } - SVSImplHandler *svs_handler = dynamic_cast(handler.get()); if (!svs_handler) { throw std::logic_error("Failed to cast to SVSImplHandler"); } - { - std::lock_guard lock(this->pimplGuard_); - this->impl_ = std::move(svs_handler->impl); - if (this->impl_) - setReady(); + + std::lock_guard replace_lock(this->implMutationGuard_); + if (getImpl()) { + throw std::logic_error("SVSIndex::setImpl called on non-empty impl_"); } + + std::lock_guard lock(this->pimplGuard_); + this->impl_ = std::move(svs_handler->impl); } // Assuming parallelism was updated to reflect the number of available threads before this @@ -290,29 +296,27 @@ class SVSIndex : public VecSimIndexAbstract, fl auto points = svs::data::SimpleDataView{typed_vectors_data, n, this->dim}; if constexpr (!isMulti) { - if (ready()) { - // SVS index does not support overriding vectors with the same label - // so we have to delete them first if needed - deleted_num = deleteVectorsImpl(labels, n); - } + // SVS index does not support overriding vectors with the same label + // so we have to delete them first if needed + deleted_num = deleteVectorsImpl(labels, n); } - this->pimplGuard_.lock_shared(); - if (!ready()) { - this->pimplGuard_.unlock_shared(); - // SVS index instance cannot be empty, so we have to construct it at first rows - std::lock_guard lock(this->pimplGuard_); - if (!ready()) { - impl_ = initImpl(points, ids); - assert(this->impl_ != nullptr); - setReady(); + this->implMutationGuard_.lock_shared(); + if (auto impl = getImpl()) { + impl->add_points(points, ids, /*reuse_empty*/ false); + this->implMutationGuard_.unlock_shared(); + } else { + this->implMutationGuard_.unlock_shared(); + this->implMutationGuard_.lock(); + if (auto existing = getImpl()) { + existing->add_points(points, ids, /*reuse_empty*/ false); } else { - impl_->add_points(points, ids, /*reuse_empty*/ false); + auto built = initImpl(points, ids); + assert(built != nullptr); + std::lock_guard lock(this->pimplGuard_); + this->impl_ = std::move(built); } - } else { - // Add new points to existing SVS index - impl_->add_points(points, ids, /*reuse_empty*/ false); - this->pimplGuard_.unlock_shared(); + this->implMutationGuard_.unlock(); } return n - deleted_num; @@ -321,57 +325,54 @@ class SVSIndex : public VecSimIndexAbstract, fl void consolidate(const std::vector &labels) override { // There is documentation for consolidate(): // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.consolidate - std::shared_lock lock(this->pimplGuard_); - if (!ready()) - return; - - impl_->consolidate(labels); + if (auto impl = getImpl()) { + impl->consolidate(labels); + } } int deleteVectorImpl(const labelType label) { - if (indexLabelCount() == 0) { + auto impl = getImpl(); + if (!impl) { return 0; } - int deleted_num = 0; - { - std::shared_lock lock(this->pimplGuard_); - deleted_num = impl_->delete_entries(std::span{&label, 1}); - } - + const int deleted_num = impl->delete_entries(std::span{&label, 1}); this->markIndexUpdate(deleted_num); return deleted_num; } int deleteVectorsImpl(const labelType *labels, size_t n) { - if (indexLabelCount() == 0) { + auto impl = getImpl(); + if (!impl) { return 0; } - int deleted_num = 0; - { - std::shared_lock lock(this->pimplGuard_); - deleted_num = impl_->delete_entries(std::span{labels, n}); - } - - if (deleted_num > 0) + const int deleted_num = impl->delete_entries(std::span{labels, n}); + if (deleted_num > 0) { this->markIndexUpdate(deleted_num); - + } return deleted_num; } - // Count deletions and consolidate index if needed + // Count deletions and drop the index instance if it became empty void markIndexUpdate(size_t n = 1) { - if (!ready()) + auto impl = getImpl(); + if (!impl) { return; + } // SVS index instance should not be empty - if (indexLabelCount() == 0) { - std::lock_guard lock(this->pimplGuard_); - if (indexLabelCountUnsafe() == 0) { - setUnready(); - this->impl_.reset(); - + if (labelCountOf(*impl) == 0) { + std::lock_guard replace_lock(this->implMutationGuard_); + if (getImpl() != impl) { + // The instance was already droped + return; + } + if (labelCountOf(*impl) == 0) { + { + std::lock_guard lock(this->pimplGuard_); + this->impl_.reset(); + } num_marked_deleted.store(0, std::memory_order_relaxed); return; } @@ -380,6 +381,14 @@ class SVSIndex : public VecSimIndexAbstract, fl num_marked_deleted.fetch_add(n, std::memory_order_relaxed); } + static size_t labelCountOf(const impl_type &impl) { + if constexpr (isMulti) { + return impl.labelcount(); + } else { + return impl.size(); + } + } + bool isTwoLevelLVQ(const VecSimSvsQuantBits &qbits) { switch (qbits) { case VecSimSvsQuant_4x4: @@ -416,67 +425,29 @@ class SVSIndex : public VecSimIndexAbstract, fl ~SVSIndex() = default; - bool ready() const override { return this->impl_ready_.load(std::memory_order_acquire); } + bool ready() const override { return getImpl() != nullptr; } size_t indexSize() const override { return indexStorageSize(); } size_t indexStorageSize() const override { - std::shared_lock lock(this->pimplGuard_); - if (ready()) { - return impl_->view_data().size(); - } else { - return 0; - } + auto impl = getImpl(); + return impl ? impl->view_data().size() : 0; } size_t indexCapacity() const override { - std::shared_lock lock(this->pimplGuard_); - if (ready()) { - return storage_traits_t::storage_capacity(impl_->view_data()); - } else { - return 0; - } - } - - size_t indexLabelCountUnsafe() const { - if constexpr (isMulti) { - if (ready()) { - return impl_->labelcount(); - } else { - return 0; - } - } else { - if (ready()) { - return impl_->size(); - } else { - return 0; - } - } + auto impl = getImpl(); + return impl ? storage_traits_t::storage_capacity(impl->view_data()) : 0; } size_t indexLabelCount() const override { - if constexpr (isMulti) { - if (ready()) { - std::shared_lock lock(this->pimplGuard_); - return impl_->labelcount(); - } else { - return 0; - } - } else { - if (ready()) { - std::shared_lock lock(this->pimplGuard_); - return impl_->size(); - } else { - return 0; - } - } + auto impl = getImpl(); + return impl ? labelCountOf(*impl) : 0; } vecsim_stl::set getLabelsSet() const override { vecsim_stl::set labels(this->allocator); - if (ready()) { - std::shared_lock lock(this->pimplGuard_); - impl_->on_ids([&labels](size_t label) { labels.insert(label); }); + if (auto impl = getImpl()) { + impl->on_ids([&labels](size_t label) { labels.insert(label); }); } return labels; } @@ -633,12 +604,8 @@ class SVSIndex : public VecSimIndexAbstract, fl } bool isLabelExists(labelType label) const override { - if (ready()) { - std::shared_lock lock(this->pimplGuard_); - return impl_->has_id(label); - } else { - return false; - } + auto impl = getImpl(); + return impl ? impl->has_id(label) : false; } size_t getParallelism() const override { return threadpool_.getParallelism(); } @@ -653,16 +620,13 @@ class SVSIndex : public VecSimIndexAbstract, fl } double getDistanceFrom_Unsafe(labelType label, const void *vector_data) const override { - if (!ready()) { - return std::numeric_limits::quiet_NaN(); - } - std::shared_lock lock(this->pimplGuard_); - if (!impl_->has_id(label)) { + auto impl = getImpl(); + if (!impl || !impl->has_id(label)) { return std::numeric_limits::quiet_NaN(); } auto query_datum = std::span{static_cast(vector_data), this->dim}; - auto dist = impl_->get_distance(label, query_datum); + auto dist = impl->get_distance(label, query_datum); return toVecSimDistance(dist); } @@ -670,12 +634,17 @@ class SVSIndex : public VecSimIndexAbstract, fl VecSimQueryParams *queryParams) const override { auto rep = new VecSimQueryReply(this->allocator); this->lastMode = STANDARD_KNN; - if (k == 0 || this->indexLabelCount() == 0) { + auto impl = getImpl(); + if (k == 0 || !impl) { return rep; } { + const auto label_count = labelCountOf(*impl); + if (label_count == 0) { + return rep; + } // limit result size to index size - k = std::min(k, this->indexLabelCount()); + k = std::min(k, label_count); auto processed_query_ptr = this->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); @@ -683,14 +652,13 @@ class SVSIndex : public VecSimIndexAbstract, fl auto query = svs::data::ConstSimpleDataView{ static_cast(processed_query), 1, this->dim}; auto result = svs::QueryResult{query.size(), k}; - std::shared_lock lock(this->pimplGuard_); - auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams, + auto sp = svs_details::joinSearchParams(impl->get_search_parameters(), queryParams, is_two_level_lvq); auto timeoutCtx = queryParams ? queryParams->timeoutCtx : nullptr; auto cancel = [timeoutCtx]() { return VECSIM_TIMEOUT(timeoutCtx); }; - impl_->search(result.view(), query, sp, cancel); + impl->search(result.view(), query, sp, cancel); if (cancel()) { rep->code = VecSim_QueryReply_TimedOut; return rep; @@ -717,12 +685,11 @@ class SVSIndex : public VecSimIndexAbstract, fl VecSimQueryParams *queryParams) const override { auto rep = new VecSimQueryReply(this->allocator); this->lastMode = RANGE_QUERY; - if (radius == 0 || this->indexLabelCount() == 0) { + auto impl = getImpl(); + if (radius == 0 || !impl || labelCountOf(*impl) == 0) { return rep; } { - std::shared_lock lock(this->pimplGuard_); - auto timeoutCtx = queryParams ? queryParams->timeoutCtx : nullptr; auto cancel = [timeoutCtx]() { return VECSIM_TIMEOUT(timeoutCtx); }; @@ -733,7 +700,7 @@ class SVSIndex : public VecSimIndexAbstract, fl this->dim}; // Base search parameters for the SVS iterator schedule. - auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams, + auto sp = svs_details::joinSearchParams(impl->get_search_parameters(), queryParams, is_two_level_lvq); // SVS BatchIterator handles the search in batches // The batch size is set to the index search window size by default @@ -741,7 +708,7 @@ class SVSIndex : public VecSimIndexAbstract, fl // Create SVS BatchIterator for range search // Search result is cached in the iterator and can be accessed by the user - auto svs_it = impl_->make_batch_iterator(query); + auto svs_it = impl->make_batch_iterator(query); svs_it.next(batch_size, cancel); if (cancel()) { rep->code = VecSim_QueryReply_TimedOut; @@ -793,12 +760,14 @@ class SVSIndex : public VecSimIndexAbstract, fl // take ownership of the blob copy and pass it to the batch iterator. auto *queryBlobCopyPtr = queryBlobCopy.release(); // Ownership of queryBlobCopy moves to VecSimBatchIterator that will free it at the end. - if (indexLabelCount() == 0) { + auto impl = getImpl(); + if (!impl || labelCountOf(*impl) == 0) { return new (this->getAllocator()) NullSVS_BatchIterator(queryBlobCopyPtr, queryParams, this->getAllocator()); } else { return new (this->getAllocator()) SVS_BatchIterator( - queryBlobCopyPtr, impl_.get(), queryParams, this->getAllocator(), is_two_level_lvq); + queryBlobCopyPtr, std::move(impl), queryParams, this->getAllocator(), + is_two_level_lvq); } } @@ -832,11 +801,10 @@ class SVSIndex : public VecSimIndexAbstract, fl } void runGC() override { - if (ready()) { - std::shared_lock lock(this->pimplGuard_); + if (auto impl = getImpl()) { // There is documentation for compact(): // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.compact - impl_->compact(); + impl->compact(); } num_marked_deleted.store(0, std::memory_order_relaxed); } @@ -865,14 +833,18 @@ class SVSIndex : public VecSimIndexAbstract, fl } else { std::vector> vectors_output; + auto impl = getImpl(); + if (!impl) { + return vectors_output; + } if constexpr (isMulti) { // Multi-index case: get all vectors for this label - auto it = impl_->get_label_to_external_lookup().find(label); - if (it != impl_->get_label_to_external_lookup().end()) { + auto it = impl->get_label_to_external_lookup().find(label); + if (it != impl->get_label_to_external_lookup().end()) { const auto &external_ids = it->second; for (auto external_id : external_ids) { - auto indexed_span = impl_->get_parent_index().get_datum(external_id); + auto indexed_span = impl->get_parent_index().get_datum(external_id); // For uncompressed data, indexed_span should be a simple span const char *data_ptr = reinterpret_cast(indexed_span.data()); @@ -883,7 +855,7 @@ class SVSIndex : public VecSimIndexAbstract, fl } } else { // Single-index case - auto indexed_span = impl_->get_datum(label); + auto indexed_span = impl->get_datum(label); // For uncompressed data, indexed_span should be a simple span const char *data_ptr = reinterpret_cast(indexed_span.data()); diff --git a/src/VecSim/algorithms/svs/svs_batch_iterator.h b/src/VecSim/algorithms/svs/svs_batch_iterator.h index 116546150..a31c02397 100644 --- a/src/VecSim/algorithms/svs/svs_batch_iterator.h +++ b/src/VecSim/algorithms/svs/svs_batch_iterator.h @@ -29,6 +29,9 @@ class SVS_BatchIterator : public VecSimBatchIterator { using dist_type = typename Index::distance_type; size_t dim; + // Declared before impl_ so that impl_, which holds a raw back-pointer into the index, is + // destroyed first. + std::shared_ptr index_; impl_type impl_; typename impl_type::const_iterator curr_it; size_t batch_size; @@ -67,14 +70,16 @@ class SVS_BatchIterator : public VecSimBatchIterator { } public: - SVS_BatchIterator(void *query_vector, const Index *index, const VecSimQueryParams *queryParams, + SVS_BatchIterator(void *query_vector, std::shared_ptr index, + const VecSimQueryParams *queryParams, std::shared_ptr allocator, bool is_two_level_lvq) : VecSimBatchIterator{query_vector, queryParams ? queryParams->timeoutCtx : nullptr, std::move(allocator)}, - dim{index->dimensions()}, impl_{index->make_batch_iterator(std::span{ - static_cast(query_vector), dim})}, + dim{index->dimensions()}, index_{std::move(index)}, + impl_{index_->make_batch_iterator( + std::span{static_cast(query_vector), dim})}, curr_it{impl_.begin()} { - auto sp = svs_details::joinSearchParams(index->get_search_parameters(), queryParams, + auto sp = svs_details::joinSearchParams(index_->get_search_parameters(), queryParams, is_two_level_lvq); batch_size = queryParams && queryParams->batchSize ? queryParams->batchSize diff --git a/src/VecSim/algorithms/svs/svs_serializer_impl.h b/src/VecSim/algorithms/svs/svs_serializer_impl.h index 5e551ef81..33fc6ca53 100644 --- a/src/VecSim/algorithms/svs/svs_serializer_impl.h +++ b/src/VecSim/algorithms/svs/svs_serializer_impl.h @@ -76,7 +76,11 @@ template void SVSIndex::impl_save( const std::string &location) { - impl_->save(location + "/config", location + "/graph", location + "/data"); + auto impl = getImpl(); + if (!impl) { + throw std::runtime_error("SVSIndex::impl_save called on an empty index"); + } + impl->save(location + "/config", location + "/graph", location + "/data"); } // This function will load the serialized svs index from the given folder path @@ -92,6 +96,7 @@ void SVSIndex // Verify metadata compatibility, will throw runtime exception if not compatible compareMetadataFile(folder_path + "/metadata"); + std::shared_ptr loaded_impl; if constexpr (isMulti) { auto loaded = svs::concurrent::auto_multi_dynamic_assemble( folder_path + "/config", @@ -101,7 +106,7 @@ void SVSIndex this->getAllocator())), distance_f(), std::move(threadpool_handle), svs::concurrent::MultiMutableVamanaLoad::FROM_MULTI, logger_); - impl_ = std::make_unique(std::move(loaded)); + loaded_impl = std::make_shared(std::move(loaded)); } else { auto loaded = svs::concurrent::auto_dynamic_assemble( folder_path + "/config", @@ -110,9 +115,12 @@ void SVSIndex SVS_LAZY(storage_traits_t::load(folder_path + "/data", this->blockSize, this->dim, this->getAllocator())), distance_f(), std::move(threadpool_handle), false, logger_); - impl_ = std::make_unique(std::move(loaded)); + loaded_impl = std::make_shared(std::move(loaded)); } - setReady(); + + std::lock_guard replace_lock(this->implMutationGuard_); + std::lock_guard lock(this->pimplGuard_); + impl_ = std::move(loaded_impl); } template bool SVSIndex::checkIntegrity() const { - if (!impl_) { + auto impl = getImpl(); + if (!impl) { throw std::runtime_error( "SVSIndex integrity check failed: index implementation (impl_) is null."); } @@ -171,9 +180,9 @@ bool SVSIndex try { // SVS internal index integrity validation if constexpr (isMulti) { - impl_->get_parent_index().debug_check_invariants(true); + impl->get_parent_index().debug_check_invariants(true); } else { - impl_->debug_check_invariants(true); + impl->debug_check_invariants(true); } } // debug_check_invariants throws svs::lib::ANNException : public std::runtime_error in case of @@ -183,10 +192,10 @@ bool SVSIndex } try { - size_t index_size = impl_->size(); - size_t storage_size = impl_->view_data().size(); - size_t capacity = storage_traits_t::storage_capacity(impl_->view_data()); - size_t label_count = this->indexLabelCount(); + size_t index_size = impl->size(); + size_t storage_size = impl->view_data().size(); + size_t capacity = storage_traits_t::storage_capacity(impl->view_data()); + size_t label_count = labelCountOf(*impl); // Storage size must match index size if (storage_size != index_size) { @@ -204,7 +213,7 @@ bool SVSIndex bool label_validation_passed = true; try { - impl_->on_ids([&](size_t label) { labels_counted++; }); + impl->on_ids([&](size_t label) { labels_counted++; }); // Validate label count consistency label_validation_passed = (labels_counted == label_count); From f301d15ada7f9da8409f6992159347afe66c97b2 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 14 Sep 2026 01:32:34 -0700 Subject: [PATCH 35/64] format --- src/VecSim/algorithms/svs/svs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 7a87fcc45..cfb000957 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -125,7 +125,7 @@ class SVSIndex : public VecSimIndexAbstract, fl // initImpl() needs at least one point to compute an entry point. // so impl_ is null until the first insert std::shared_ptr impl_; - + mutable std::shared_mutex pimplGuard_; // Serialize mutators of the impl_ pointer (exclusive) // vs add_points() (shared) to avoid silent drops of additions From 93ad60cb0ebf66efbb016a226ca4812601334ed8 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 14 Sep 2026 01:48:44 -0700 Subject: [PATCH 36/64] fix leaking consolidate jobs --- src/VecSim/algorithms/svs/svs_tiered.h | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 1e1783cca..543a7105d 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -276,6 +276,9 @@ class TieredSVSIndex : public VecSimTieredIndex { idType currInvalidJobId; // A unique arbitrary identifier for accessing invalid jobs std::mutex invalidJobsLookupGuard; + vecsim_stl::unordered_set pendingConsolidateJobs; + std::mutex consolidateJobsGuard; + size_t flat_buffer_bound; /// @@ -649,6 +652,10 @@ class TieredSVSIndex : public VecSimTieredIndex { auto svs_index = index->GetSVSIndex(); svs_index->setParallelism(1); svs_index->consolidate(consolidate_job->labels); + { + std::lock_guard jobs_lock(index->consolidateJobsGuard); + index->pendingConsolidateJobs.erase(job); + } delete job; } @@ -698,6 +705,10 @@ class TieredSVSIndex : public VecSimTieredIndex { AsyncJob *new_consolidate_job = new (this->allocator) SVSConsolidateJob(this->allocator, {label}, SVSIndexConsolidateWrapper, this); + { + std::lock_guard lock(this->consolidateJobsGuard); + this->pendingConsolidateJobs.insert(new_consolidate_job); + } // Insert job to the queue. this->submitSingleJob(new_consolidate_job); } @@ -860,7 +871,8 @@ class TieredSVSIndex : public VecSimTieredIndex { std::shared_ptr allocator) : Base(svs_index, bf_index, tiered_index_params, allocator), uncompletedJobs(this->allocator), labelToInsertJobs(this->allocator), - invalidJobs(this->allocator), currInvalidJobId(0) { + invalidJobs(this->allocator), currInvalidJobId(0), + pendingConsolidateJobs(this->allocator) { const auto &tiered_svs_params = tiered_index_params.specificParams.tieredSVSParams; // If flatBufferLimit is not initialized (0), use the default update threshold. @@ -900,6 +912,11 @@ class TieredSVSIndex : public VecSimTieredIndex { for (auto &it : this->invalidJobs) { delete it.second; } + + // Delete all the pending consolidate jobs + for (auto *job : this->pendingConsolidateJobs) { + delete job; + } } int addVector(const void *blob, labelType label) override { From f34be249be458696945a733205d0782aaace5d5c Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 14 Sep 2026 02:32:02 -0700 Subject: [PATCH 37/64] resubmit insert job if backend isn't ready --- src/VecSim/algorithms/svs/svs_tiered.h | 28 ++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 543a7105d..1a5f3d133 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -597,9 +597,17 @@ class TieredSVSIndex : public VecSimTieredIndex { static void executeInsertJobWrapper(AsyncJob *job) { auto *insert_job = static_cast(job); auto *job_index = static_cast *>(insert_job->index); - // prevent parallel execution with index initilizing job - std::shared_lock lock(job_index->updateJobMutex); - job_index->executeInsertJob(insert_job); + InsertJobOutcome outcome; + { + // prevent parallel execution with index initilizing job + std::shared_lock lock(job_index->updateJobMutex); + outcome = job_index->executeInsertJob(insert_job); + } + + if (outcome == InsertJobOutcome::Deferred) { + job_index->submitSingleJob(job); + return; + } delete job; } @@ -727,9 +735,10 @@ class TieredSVSIndex : public VecSimTieredIndex { return curInvalidId; } - void executeInsertJob(SVSInsertJob *job) { + enum class InsertJobOutcome { Completed, Deferred }; + + InsertJobOutcome executeInsertJob(SVSInsertJob *job) { auto svs_index = GetSVSIndex(); - assert(svs_index->ready()); // Note that accessing the job fields should occur with flat index guard held (here and // later). @@ -741,7 +750,13 @@ class TieredSVSIndex : public VecSimTieredIndex { this->invalidJobsLookupGuard.lock(); this->invalidJobs.erase(job->id); this->invalidJobsLookupGuard.unlock(); - return; + return InsertJobOutcome::Completed; + } + + // Backend is still initilizing. Resubmit the job + if (!svs_index->ready() && this->indexUpdateScheduled.test()) { + this->flatIndexGuard.unlock_shared(); + return InsertJobOutcome::Deferred; } // Copy the vector blob out of the flat buffer while holding flatIndexGuard, so we @@ -794,6 +809,7 @@ class TieredSVSIndex : public VecSimTieredIndex { this->invalidJobsLookupGuard.unlock(); } this->flatIndexGuard.unlock(); + return InsertJobOutcome::Completed; } void initSVSIndex(size_t availableThreads) { From 86feaa660daa0ec852d86f760a3b16fb6efd5ec1 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Mon, 14 Sep 2026 05:34:05 -0700 Subject: [PATCH 38/64] use RAII lock to harmonize topKQuery and rangeQuery --- src/VecSim/algorithms/svs/svs_tiered.h | 85 ++------------------------ src/VecSim/vec_sim_tiered_index.h | 37 +++++++---- 2 files changed, 30 insertions(+), 92 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 1a5f3d133..75a54d706 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -568,6 +568,12 @@ class TieredSVSIndex : public VecSimTieredIndex { } #endif +protected: + std::shared_lock lockMainIndexForQuery() const override { + // returns a no-op lock because SVS does its locking internally + return std::shared_lock{}; + } + private: /** * @brief Init the SVS index in a thread-safe manner. @@ -1270,85 +1276,6 @@ class TieredSVSIndex : public VecSimTieredIndex { return infoIterator; } - VecSimQueryReply *topKQuery(const void *queryBlob, size_t k, - VecSimQueryParams *queryParams) const override { - // SVS handles its own internal locking for concurrent search + modification, - // so we don't need mainIndexGuard for backend queries. - this->flatIndexGuard.lock_shared(); - - if (this->frontendIndex->indexSize() == 0) { - this->flatIndexGuard.unlock_shared(); - - auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); - const void *processed_query = processed_query_ptr.get(); - return this->backendIndex->topKQuery(processed_query, k, queryParams); - } else { - auto flat_results = this->frontendIndex->topKQuery(queryBlob, k, queryParams); - this->flatIndexGuard.unlock_shared(); - - if (flat_results->code != VecSim_QueryReply_OK) { - assert(flat_results->results.empty()); - return flat_results; - } - - auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); - const void *processed_query = processed_query_ptr.get(); - auto main_results = this->backendIndex->topKQuery(processed_query, k, queryParams); - - if (main_results->code != VecSim_QueryReply_OK) { - VecSimQueryReply_Free(flat_results); - assert(main_results->results.empty()); - return main_results; - } - - return merge_result_lists(main_results, flat_results, k); - } - } - - VecSimQueryReply *rangeQuery(const void *queryBlob, double radius, - VecSimQueryParams *queryParams, - VecSimQueryReply_Order order) const override { - // SVS handles its own internal locking for concurrent search + modification, - // so we don't need mainIndexGuard for backend queries. - this->flatIndexGuard.lock_shared(); - - if (this->frontendIndex->indexSize() == 0) { - this->flatIndexGuard.unlock_shared(); - - auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); - const void *processed_query = processed_query_ptr.get(); - auto res = this->backendIndex->rangeQuery(processed_query, radius, queryParams); - sort_results(res, order); - return res; - } else { - auto flat_results = this->frontendIndex->rangeQuery(queryBlob, radius, queryParams); - this->flatIndexGuard.unlock_shared(); - - if (flat_results->code != VecSim_QueryReply_OK) { - return flat_results; - } - - auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); - const void *processed_query = processed_query_ptr.get(); - auto main_results = - this->backendIndex->rangeQuery(processed_query, radius, queryParams); - - if (BY_SCORE == order) { - sort_results_by_score_then_id(main_results); - sort_results_by_score_then_id(flat_results); - - auto code = main_results->code; - VecSimQueryReply *ret = merge_result_lists(main_results, flat_results, -1); - ret->code = code; - return ret; - } else { // BY_ID - concat_results(main_results, flat_results); - filter_results_by_id(main_results); - return main_results; - } - } - } - VecSimBatchIterator *newBatchIterator(const void *queryBlob, VecSimQueryParams *queryParams) const override { // The query blob will be processed and copied by the internal indexes's batch iterator. diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 1497ffba3..8b2df74a0 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -46,6 +46,11 @@ struct AsyncJob : public VecsimBaseObject { template class VecSimTieredIndex : public VecSimIndexInterface { protected: + // RAII acquisition of mainIndexGuard to allow different locking behavior in child classes + virtual std::shared_lock lockMainIndexForQuery() const { + return std::shared_lock(mainIndexGuard); + } + VecSimIndexAbstract *backendIndex; BruteForceIndex *frontendIndex; @@ -237,9 +242,8 @@ VecSimTieredIndex::topKQueryImp(const void *queryBlob, size_ // Simply query the main index and return the results while holding the lock. auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); - this->mainIndexGuard.lock_shared(); + auto mainLock = lockMainIndexForQuery(); auto res = this->backendIndex->topKQuery(processed_query, k, queryParams); - this->mainIndexGuard.unlock_shared(); return res; } else { @@ -256,10 +260,12 @@ VecSimTieredIndex::topKQueryImp(const void *queryBlob, size_ auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); - // Lock the main index and query it. - this->mainIndexGuard.lock_shared(); - auto main_results = this->backendIndex->topKQuery(processed_query, k, queryParams); - this->mainIndexGuard.unlock_shared(); + VecSimQueryReply *main_results; + { + // Lock the main index and query it. + auto mainLock = lockMainIndexForQuery(); + main_results = this->backendIndex->topKQuery(processed_query, k, queryParams); + } // If the query failed (currently only on timeout), return the error code. if (main_results->code != VecSim_QueryReply_OK) { @@ -302,10 +308,12 @@ VecSimTieredIndex::rangeQueryImp(const void *queryBlob, doub auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); - // Simply query the main index and return the results while holding the lock. - this->mainIndexGuard.lock_shared(); - auto res = this->backendIndex->rangeQuery(processed_query, radius, queryParams); - this->mainIndexGuard.unlock_shared(); + VecSimQueryReply *res; + { + auto mainLock = lockMainIndexForQuery(); + // Simply query the main index and return the results while holding the lock. + res = this->backendIndex->rangeQuery(processed_query, radius, queryParams); + } // We could have passed the order to the main index, but we can sort them here after // unlocking it instead. @@ -326,9 +334,12 @@ VecSimTieredIndex::rangeQueryImp(const void *queryBlob, doub auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); // Lock the main index and query it. - this->mainIndexGuard.lock_shared(); - auto main_results = this->backendIndex->rangeQuery(processed_query, radius, queryParams); - this->mainIndexGuard.unlock_shared(); + + VecSimQueryReply *main_results; + { + auto mainLock = lockMainIndexForQuery(); + main_results = this->backendIndex->rangeQuery(processed_query, radius, queryParams); + } // Merge the results and return, avoiding duplicates. // At this point, the return code of the FLAT index is OK, and the return code of the MAIN From 2ff7cf201cad0e8795a13ca2c7784e68b997b7d1 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 14 Sep 2026 06:25:27 -0700 Subject: [PATCH 39/64] fix and optimize --- src/VecSim/algorithms/svs/svs.h | 17 +++++++---------- src/python_bindings/bindings.cpp | 4 +++- tests/flow/test_svs_tiered.py | 5 +++++ 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index cfb000957..40b63dae0 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -338,7 +338,7 @@ class SVSIndex : public VecSimIndexAbstract, fl } const int deleted_num = impl->delete_entries(std::span{&label, 1}); - this->markIndexUpdate(deleted_num); + this->markIndexUpdate(impl, deleted_num); return deleted_num; } @@ -350,18 +350,12 @@ class SVSIndex : public VecSimIndexAbstract, fl const int deleted_num = impl->delete_entries(std::span{labels, n}); if (deleted_num > 0) { - this->markIndexUpdate(deleted_num); + this->markIndexUpdate(impl, deleted_num); } return deleted_num; } - // Count deletions and drop the index instance if it became empty - void markIndexUpdate(size_t n = 1) { - auto impl = getImpl(); - if (!impl) { - return; - } - + void markIndexUpdate(const std::shared_ptr &impl, size_t n = 1) { // SVS index instance should not be empty if (labelCountOf(*impl) == 0) { std::lock_guard replace_lock(this->implMutationGuard_); @@ -426,7 +420,10 @@ class SVSIndex : public VecSimIndexAbstract, fl ~SVSIndex() = default; - bool ready() const override { return getImpl() != nullptr; } + bool ready() const override { + std::shared_lock lock(this->pimplGuard_); + return this->impl_ != nullptr; + } size_t indexSize() const override { return indexStorageSize(); } diff --git a/src/python_bindings/bindings.cpp b/src/python_bindings/bindings.cpp index b68c14653..5888f5d5c 100644 --- a/src/python_bindings/bindings.cpp +++ b/src/python_bindings/bindings.cpp @@ -214,7 +214,9 @@ class PyVecSimIndex { return PyBatchIterator(index, py_batch_ptr); } - void runGC() { VecSimTieredIndex_GC(index.get()); } + // Not VecSimTieredIndex_GC(): that is a no-op unless the index is tiered, while + // SVSIndex::runGC() compacts and rebuilds reverse edges on a plain index too. + void runGC() { index->runGC(); } py::object getVector(labelType label) { VecSimIndexBasicInfo info = index->basicInfo(); diff --git a/tests/flow/test_svs_tiered.py b/tests/flow/test_svs_tiered.py index 746bab43f..e0d8b7f8d 100644 --- a/tests/flow/test_svs_tiered.py +++ b/tests/flow/test_svs_tiered.py @@ -166,6 +166,9 @@ def create_tiered_index(test_logger, is_multi: bool, num_per_label=1, data_type= test_logger.info(f"Total time for inserting vectors to the tiered index and indexing them into SVS using {threads_num}" f" threads took {round_ms(tiered_index_time)} ms") + index.run_gc() + index.wait_for_index() + # Measure total memory of the tiered index. tiered_memory = bytes_to_mega(index.index_memory()) @@ -175,6 +178,8 @@ def create_tiered_index(test_logger, is_multi: bool, num_per_label=1, data_type= _, svs_index_time, _ = indices_ctx.populate_index(svs_index) test_logger.info(f"Insert {num_elements} vectors directly to SVS index (one by one) took {round_(svs_index_time)} s") + + svs_index.run_gc() svs_memory = bytes_to_mega(svs_index.index_memory()) test_logger.info(f"total memory of svs index = {svs_memory} MB") From ded890c2671ff20b2fc7f7eb43b449592f53dba2 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Mon, 14 Sep 2026 06:47:58 -0700 Subject: [PATCH 40/64] generalize SharedMutexLockable and harominze SVS/HNSW functions with multiple locks --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 36 +++++++++++++----------- src/VecSim/algorithms/svs/svs_tiered.h | 22 ++++++--------- src/VecSim/vec_sim_tiered_index.h | 28 ++++++++++++++---- 3 files changed, 51 insertions(+), 35 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index 11f6cf76f..bc25b355e 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -154,6 +154,25 @@ class TieredHNSWIndex : public VecSimTieredIndex { #include "VecSim/algorithms/hnsw/hnsw_tiered_tests_friends.h" #endif +protected: + // Adapts HNSW's own data-guard mutex to Lockable, for indexSize/indexCapacity locking. + class HnswDataGuardLockable : public Lockable { + HNSWIndex *hnsw; + + public: + explicit HnswDataGuardLockable(HNSWIndex *hnsw) : hnsw(hnsw) {} + void lock() const override { hnsw->lockSharedIndexDataGuard(); } + void unlock() const override { hnsw->unlockSharedIndexDataGuard(); } + }; + HnswDataGuardLockable hnswDataGuardLockable{this->getHNSWIndex()}; + + ScopedLocks lockIndexForSize() const override { + return ScopedLocks(this->flatIndexLockable, hnswDataGuardLockable); + } + + // Old behavior: indexCapacity() took no lock at all. + ScopedLocks lockIndexForCapacity() const override { return ScopedLocks(); } + public: class TieredHNSW_BatchIterator : public VecSimBatchIterator { private: @@ -217,8 +236,6 @@ class TieredHNSWIndex : public VecSimTieredIndex { size_t getNumMarkedDeleted() const override { return this->getHNSWIndex()->getNumMarkedDeleted(); } - size_t indexSize() const override; - size_t indexCapacity() const override; double getDistanceFrom_Unsafe(labelType label, const void *blob) const override; // Do nothing here, each tier (flat buffer and HNSW) should increase capacity for itself when // needed. @@ -738,21 +755,6 @@ TieredHNSWIndex::~TieredHNSWIndex() { } } -template -size_t TieredHNSWIndex::indexSize() const { - this->flatIndexGuard.lock_shared(); - this->getHNSWIndex()->lockSharedIndexDataGuard(); - size_t res = this->backendIndex->indexSize() + this->frontendIndex->indexSize(); - this->getHNSWIndex()->unlockSharedIndexDataGuard(); - this->flatIndexGuard.unlock_shared(); - return res; -} - -template -size_t TieredHNSWIndex::indexCapacity() const { - return this->backendIndex->indexCapacity() + this->frontendIndex->indexCapacity(); -} - // In the tiered index, we assume that the blobs are processed by the flat buffer // before being transferred to the HNSW index. // When inserting vectors directly into the HNSW index—such as in VecSim_WriteInPlace mode— or when diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 75a54d706..8db0d6676 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -569,9 +569,15 @@ class TieredSVSIndex : public VecSimTieredIndex { #endif protected: - std::shared_lock lockMainIndexForQuery() const override { - // returns a no-op lock because SVS does its locking internally - return std::shared_lock{}; + ScopedLocks lockMainIndexForQuery() const override { + // No-op: SVS does its query locking internally. + return ScopedLocks(); + } + + ScopedLocks lockIndexForSize() const override { return ScopedLocks(this->flatIndexLockable); } + + ScopedLocks lockIndexForCapacity() const override { + return ScopedLocks(this->flatIndexLockable); } private: @@ -1190,16 +1196,6 @@ class TieredSVSIndex : public VecSimTieredIndex { return this->GetSVSIndex()->getNumMarkedDeleted(); } - size_t indexSize() const override { - std::shared_lock flat_lock(this->flatIndexGuard); - return this->frontendIndex->indexSize() + this->backendIndex->indexSize(); - } - - size_t indexCapacity() const override { - std::shared_lock flat_lock(this->flatIndexGuard); - return this->frontendIndex->indexCapacity() + this->backendIndex->indexCapacity(); - } - double getDistanceFrom_Unsafe(labelType label, const void *blob) const override { // Try to get the distance from the flat buffer. // If the label doesn't exist, the distance will be NaN. diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 8b2df74a0..a9836e3ab 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -15,6 +15,7 @@ #include "VecSim/tombstone_interface.h" #include "VecSim/utils/query_result_utils.h" #include "VecSim/utils/alignment.h" +#include "VecSim/utils/scoped_locks.h" #include @@ -46,11 +47,6 @@ struct AsyncJob : public VecsimBaseObject { template class VecSimTieredIndex : public VecSimIndexInterface { protected: - // RAII acquisition of mainIndexGuard to allow different locking behavior in child classes - virtual std::shared_lock lockMainIndexForQuery() const { - return std::shared_lock(mainIndexGuard); - } - VecSimIndexAbstract *backendIndex; BruteForceIndex *frontendIndex; @@ -60,6 +56,18 @@ class VecSimTieredIndex : public VecSimIndexInterface { mutable std::shared_mutex flatIndexGuard; mutable std::shared_mutex mainIndexGuard; + SharedMutexLockable flatIndexLockable{flatIndexGuard}; + SharedMutexLockable mainIndexLockable{mainIndexGuard}; + + // Locking behavior for topKQuery/rangeQuery: real lock by default, overridden to a no-op + // by backends (e.g. SVS) that handle their own concurrency internally. + virtual ScopedLocks lockMainIndexForQuery() const { return ScopedLocks(mainIndexLockable); } + + // Locking behavior for indexSize/indexCapacity, which each backend defines to match + // whatever it actually needs to guard those counters with -- the two differ per backend. + virtual ScopedLocks lockIndexForSize() const = 0; + virtual ScopedLocks lockIndexForCapacity() const = 0; + void lockMainIndexGuard() const { mainIndexGuard.lock(); #ifdef BUILD_TESTS @@ -197,6 +205,16 @@ class VecSimTieredIndex : public VecSimIndexInterface { VecSimQueryParams *queryParams, VecSimQueryReply_Order order) const override; + size_t indexSize() const override { + auto locks = lockIndexForSize(); + return this->frontendIndex->indexSize() + this->backendIndex->indexSize(); + } + + size_t indexCapacity() const override { + auto locks = lockIndexForCapacity(); + return this->frontendIndex->indexCapacity() + this->backendIndex->indexCapacity(); + } + virtual inline uint64_t getAllocationSize() const override { return this->allocator->getAllocationSize() + this->backendIndex->getAllocationSize() + this->frontendIndex->getAllocationSize(); From 468e6eff7b13d8a844a1147124bb10be4238a484 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 14 Sep 2026 06:56:48 -0700 Subject: [PATCH 41/64] move droping of gc schdule flag --- src/VecSim/algorithms/svs/svs_tiered.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 1a5f3d133..ea262cbb8 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -643,13 +643,13 @@ class TieredSVSIndex : public VecSimTieredIndex { index->executeTracingCallback("GCJob::before_run_gc"); std::lock_guard lock(index->updateJobMutex); + // Release the scheduled flag to allow scheduling again + index->indexGCScheduled.clear(); + svs_index->setParallelism(std::min(availableThreads, index->backendIndex->indexSize())); // VecSimIndexAbstract::runGC() is protected static_cast(index->backendIndex)->runGC(); svs_index->setParallelism(1); - - // Release the scheduled flag to allow scheduling again - index->indexGCScheduled.clear(); } static void SVSIndexConsolidateWrapper(AsyncJob *job) { From 9050eb249a5cf38a597ebc9bbcabe11660fc730d Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Mon, 14 Sep 2026 06:57:37 -0700 Subject: [PATCH 42/64] cleanup comments --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 287 ++++++++++++----------- src/VecSim/vec_sim_tiered_index.h | 10 +- 2 files changed, 160 insertions(+), 137 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index bc25b355e..0b3f3d968 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -155,7 +155,7 @@ class TieredHNSWIndex : public VecSimTieredIndex { #endif protected: - // Adapts HNSW's own data-guard mutex to Lockable, for indexSize/indexCapacity locking. + // Adapts HNSW's own data-guard mutex to Lockable class HnswDataGuardLockable : public Lockable { HNSWIndex *hnsw; @@ -166,12 +166,16 @@ class TieredHNSWIndex : public VecSimTieredIndex { }; HnswDataGuardLockable hnswDataGuardLockable{this->getHNSWIndex()}; + ScopedLocks lockMainIndexForQuery() { return ScopedLocks(mainIndexLockable); } + ScopedLocks lockIndexForSize() const override { return ScopedLocks(this->flatIndexLockable, hnswDataGuardLockable); } - // Old behavior: indexCapacity() took no lock at all. - ScopedLocks lockIndexForCapacity() const override { return ScopedLocks(); } + ScopedLocks lockIndexForCapacity() const override { + // No-op: No locking required for capacity() + return ScopedLocks(); + } public: class TieredHNSW_BatchIterator : public VecSimBatchIterator { @@ -185,16 +189,15 @@ class TieredHNSWIndex : public VecSimTieredIndex { VecSimBatchIterator *flat_iterator; VecSimBatchIterator *hnsw_iterator; - // On single value indices, this set holds the IDs of the results that were returned from - // the flat buffer. - // On multi value indices, this set holds the IDs of all the results that were returned. - // The difference between the two cases is that on multi value indices, the same ID can - // appear in both indexes and results with different scores, and therefore we can't tell in - // advance when we expect a possibility of a duplicate. - // On single value indices, a duplicate may appear at the same batch (and we will handle it - // when merging the results) Or it may appear in a different batches, first from the flat - // buffer and then from the HNSW, in the cases where a better result if found later in HNSW - // because of the approximate nature of the algorithm. + // On single value indices, this set holds the IDs of the results that were returned + // from the flat buffer. On multi value indices, this set holds the IDs of all the + // results that were returned. The difference between the two cases is that on multi + // value indices, the same ID can appear in both indexes and results with different + // scores, and therefore we can't tell in advance when we expect a possibility of a + // duplicate. On single value indices, a duplicate may appear at the same batch (and we + // will handle it when merging the results) Or it may appear in a different batches, + // first from the flat buffer and then from the HNSW, in the cases where a better result + // if found later in HNSW because of the approximate nature of the algorithm. vecsim_stl::unordered_set returned_results_set; private: @@ -237,8 +240,8 @@ class TieredHNSWIndex : public VecSimTieredIndex { return this->getHNSWIndex()->getNumMarkedDeleted(); } double getDistanceFrom_Unsafe(labelType label, const void *blob) const override; - // Do nothing here, each tier (flat buffer and HNSW) should increase capacity for itself when - // needed. + // Do nothing here, each tier (flat buffer and HNSW) should increase capacity for itself + // when needed. VecSimIndexDebugInfo debugInfo() const override; VecSimIndexBasicInfo basicInfo() const override; VecSimIndexStatsInfo statisticInfo() const override; @@ -320,16 +323,16 @@ void TieredHNSWIndex::fixJobsAfterSwap( idToRepairJobs.insert({deleted_id, idToRepairJobs.at(prev_last_id)}); idToRepairJobs.erase(prev_last_id); } - // Update the swap jobs if the last id also needs a swap, otherwise just collect to deleted id - // to be removed from the swap jobs. + // Update the swap jobs if the last id also needs a swap, otherwise just collect to deleted + // id to be removed from the swap jobs. if (prev_last_id != deleted_id && idToSwapJob.find(prev_last_id) != idToSwapJob.end() && std::find(idsToRemove.begin(), idsToRemove.end(), prev_last_id) == idsToRemove.end()) { - // Update the curr_last_id pending swap job id after the removal that renamed curr_last_id - // with the deleted id. + // Update the curr_last_id pending swap job id after the removal that renamed + // curr_last_id with the deleted id. idsToRemove.push_back(prev_last_id); idToSwapJob.at(prev_last_id)->deleted_id = deleted_id; - // If id was deleted in-place and there is no swap job for it, this will create a new entry - // in idToSwapJob for the swapped id, otherwise it will update the existing entry. + // If id was deleted in-place and there is no swap job for it, this will create a new + // entry in idToSwapJob for the swapped id, otherwise it will update the existing entry. idToSwapJob[deleted_id] = idToSwapJob.at(prev_last_id); } else { idsToRemove.push_back(deleted_id); @@ -338,7 +341,8 @@ void TieredHNSWIndex::fixJobsAfterSwap( template void TieredHNSWIndex::invalidateRepairJobs(idType deleted_id) { - // Invalidate repair jobs for the disposed id (if exist), and update the associated swap jobs. + // Invalidate repair jobs for the disposed id (if exist), and update the associated swap + // jobs. if (idToRepairJobs.find(deleted_id) == idToRepairJobs.end()) { return; } @@ -453,9 +457,9 @@ int TieredHNSWIndex::deleteLabelFromHNSW(labelType label) { this->idToRepairJobsGuard.unlock(); if (incomingEdges.size() == 0) { - // No repair job will ever run for this element, so this is already the point at which - // it can be taken out of the graph (outside the repair jobs guard, as isolating takes - // the per-element links locks). + // No repair job will ever run for this element, so this is already the point at + // which it can be taken out of the graph (outside the repair jobs guard, as + // isolating takes the per-element links locks). this->isolateRepairedElement(id); } @@ -495,8 +499,8 @@ void TieredHNSWIndex::insertVectorToHNSW( const void *processed_storage_blob = processed_blobs.getStorageBlob(); const void *processed_for_index = processed_blobs.getQueryBlob(); - // Acquire the index data lock, so we know what is the exact index size at this time. Acquire - // the main r/w lock before to avoid deadlocks. + // Acquire the index data lock, so we know what is the exact index size at this time. + // Acquire the main r/w lock before to avoid deadlocks. this->mainIndexGuard.lock_shared(); hnsw_index->lockIndexDataGuard(); // Check if resizing is needed for HNSW index (requires write lock). @@ -507,11 +511,11 @@ void TieredHNSWIndex::insertVectorToHNSW( this->lockMainIndexGuard(); hnsw_index->lockIndexDataGuard(); - // Hold the index data lock while we store the new element. If the new node's max level is - // higher than the current one, hold the lock through the entire insertion to ensure that - // graph scans will not occur, as they will try access the entry point's neighbors. - // If an index resize is still needed, `storeNewElement` will perform it. This is OK since - // we hold the main index lock for exclusive access. + // Hold the index data lock while we store the new element. If the new node's max level + // is higher than the current one, hold the lock through the entire insertion to ensure + // that graph scans will not occur, as they will try access the entry point's neighbors. + // If an index resize is still needed, `storeNewElement` will perform it. This is OK + // since we hold the main index lock for exclusive access. auto state = hnsw_index->storeNewElement(label, processed_storage_blob); if constexpr (releaseFlatGuard) { this->flatIndexGuard.unlock_shared(); @@ -520,11 +524,13 @@ void TieredHNSWIndex::insertVectorToHNSW( // If we're still holding the index data guard, we cannot take the main index lock for // shared ownership as it may cause deadlocks, and we also cannot release the main index // lock between, since we cannot allow swap jobs to happen, as they will make the - // saved state invalid. Hence, we insert the vector with the current exclusive lock held. + // saved state invalid. Hence, we insert the vector with the current exclusive lock + // held. if (state.elementMaxLevel <= state.currMaxLevel) { hnsw_index->unlockIndexDataGuard(); } - // Take the vector from the flat buffer and insert it to HNSW (overwrite should not occur). + // Take the vector from the flat buffer and insert it to HNSW (overwrite should not + // occur). hnsw_index->indexVector(processed_for_index, label, state); if (state.elementMaxLevel > state.currMaxLevel) { hnsw_index->unlockIndexDataGuard(); @@ -532,11 +538,11 @@ void TieredHNSWIndex::insertVectorToHNSW( this->unlockMainIndexGuard(); } else { // Do the same as above except for changing the capacity, but with *shared* lock held: - // Hold the index data lock while we store the new element. If the new node's max level is - // higher than the current one, hold the lock through the entire insertion to ensure that - // graph scans will not occur, as they will try access the entry point's neighbors. - // At this point we are certain that the index has enough capacity for the new element, and - // this call will not resize the index. + // Hold the index data lock while we store the new element. If the new node's max level + // is higher than the current one, hold the lock through the entire insertion to ensure + // that graph scans will not occur, as they will try access the entry point's neighbors. + // At this point we are certain that the index has enough capacity for the new element, + // and this call will not resize the index. auto state = hnsw_index->storeNewElement(label, processed_storage_blob); if constexpr (releaseFlatGuard) { this->flatIndexGuard.unlock_shared(); @@ -545,7 +551,8 @@ void TieredHNSWIndex::insertVectorToHNSW( if (state.elementMaxLevel <= state.currMaxLevel) { hnsw_index->unlockIndexDataGuard(); } - // Take the vector from the flat buffer and insert it to HNSW (overwrite should not occur). + // Take the vector from the flat buffer and insert it to HNSW (overwrite should not + // occur). hnsw_index->indexVector(processed_for_index, label, state); if (state.elementMaxLevel > state.currMaxLevel) { hnsw_index->unlockIndexDataGuard(); @@ -591,7 +598,8 @@ int TieredHNSWIndex::deleteLabelFromHNSWInplace(labelType la /******************** Job's callbacks **********************************/ template void TieredHNSWIndex::executeInsertJob(HNSWInsertJob *job) { - // Note that accessing the job fields should occur with flat index guard held (here and later). + // Note that accessing the job fields should occur with flat index guard held (here and + // later). this->flatIndexGuard.lock_shared(); if (!job->isValid) { this->flatIndexGuard.unlock_shared(); @@ -630,20 +638,22 @@ void TieredHNSWIndex::executeInsertJob(HNSWInsertJob *job) { if (labelToInsertJobs.at(job->label).empty()) { labelToInsertJobs.erase(job->label); } - // Remove the vector from the flat buffer. This may cause the last vector id to swap with - // the deleted id. Hold the label for the last id, so we can later on update its + // Remove the vector from the flat buffer. This may cause the last vector id to swap + // with the deleted id. Hold the label for the last id, so we can later on update its // corresponding job id. Note that after calling deleteVectorById, the last id's label // shouldn't be available, since it is removed from the lookup. labelType last_vec_label = this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); int deleted = this->frontendIndex->deleteVectorById(job->label, job->id); if (deleted && job->id != this->frontendIndex->indexSize()) { - // If the vector removal caused a swap with the last id, update the relevant insert job. + // If the vector removal caused a swap with the last id, update the relevant insert + // job. this->updateInsertJobInternalId(this->frontendIndex->indexSize(), job->id, last_vec_label); } } else { - // Remove the current job from the invalid jobs' lookup, as we are about to delete it now. + // Remove the current job from the invalid jobs' lookup, as we are about to delete it + // now. this->invalidJobsLookupGuard.lock(); this->invalidJobs.erase(job->id); this->invalidJobsLookupGuard.unlock(); @@ -665,10 +675,10 @@ void TieredHNSWIndex::executeRepairJob(HNSWRepairJob *job) { } HNSWIndex *hnsw_index = this->getHNSWIndex(); - // Remove this job pointer from the repair jobs lookup BEFORE it has been executed. Had we done - // it after executing the repair job, we might have see that there is a pending repair job for - // this node id upon deleting another neighbor of this node, and we may avoid creating another - // repair job even though *it has already been executed*. + // Remove this job pointer from the repair jobs lookup BEFORE it has been executed. Had we + // done it after executing the repair job, we might have see that there is a pending repair + // job for this node id upon deleting another neighbor of this node, and we may avoid + // creating another repair job even though *it has already been executed*. this->idToRepairJobsGuard.lock(); auto &repair_jobs = this->idToRepairJobs.at(job->node_id); assert(repair_jobs.size() > 0); @@ -701,9 +711,10 @@ void TieredHNSWIndex::executeRepairJob(HNSWRepairJob *job) { } this->idToRepairJobsGuard.unlock(); - // These deleted elements have no pending repair job left, so nothing points to them anymore. - // Take them out of the graph entirely, leaving no edge in or out for the swap job to deal with. - // Done outside the repair jobs guard, as isolating takes the per-element links locks. + // These deleted elements have no pending repair job left, so nothing points to them + // anymore. Take them out of the graph entirely, leaving no edge in or out for the swap job + // to deal with. Done outside the repair jobs guard, as isolating takes the per-element + // links locks. for (idType deleted_id : fully_repaired_ids) { this->isolateRepairedElement(deleted_id); } @@ -757,9 +768,9 @@ TieredHNSWIndex::~TieredHNSWIndex() { // In the tiered index, we assume that the blobs are processed by the flat buffer // before being transferred to the HNSW index. -// When inserting vectors directly into the HNSW index—such as in VecSim_WriteInPlace mode— or when -// the flat buffer is full, we must manually preprocess the blob according to the **frontend** index -// parameters. +// When inserting vectors directly into the HNSW index—such as in VecSim_WriteInPlace mode— or +// when the flat buffer is full, we must manually preprocess the blob according to the +// **frontend** index parameters. template int TieredHNSWIndex::addVector(const void *blob, labelType label) { int ret = 1; @@ -767,7 +778,8 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l // writeMode is not protected since it is assumed to be called only from the "main thread" // (that is the thread that is exclusively calling add/delete vector). if (this->getWriteMode() == VecSim_WriteInPlace) { - // First, check if we need to overwrite the vector in-place for single (from both indexes). + // First, check if we need to overwrite the vector in-place for single (from both + // indexes). if (!this->backendIndex->isMultiValue()) { ret -= this->deleteVector(label); } @@ -787,16 +799,15 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l if (this->frontendIndex->indexSize() >= this->flatBufferLimit) { // Handle overwrite situation. if (!this->backendIndex->isMultiValue()) { - // This will do nothing (and return 0) if this label doesn't exist. Otherwise, it may - // remove vector from the flat buffer and/or the HNSW index. + // This will do nothing (and return 0) if this label doesn't exist. Otherwise, it + // may remove vector from the flat buffer and/or the HNSW index. ret -= this->deleteVector(label); } if (this->frontendIndex->indexSize() >= this->flatBufferLimit) { - // We didn't remove a vector from flat buffer due to overwrite, insert the new vector - // directly to HNSW. Since flat buffer guard was not held, no need to release it - // internally. - // Use the frontend parameters to manually prepare the blob for its transfer to the HNSW - // index. + // We didn't remove a vector from flat buffer due to overwrite, insert the new + // vector directly to HNSW. Since flat buffer guard was not held, no need to release + // it internally. Use the frontend parameters to manually prepare the blob for its + // transfer to the HNSW index. auto storage_blob = this->frontendIndex->preprocessForStorage(blob); this->insertVectorToHNSW(hnsw_index, label, storage_blob.get()); // Track direct insertion to HNSW (flat buffer was full) @@ -804,7 +815,8 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l return ret; } // Otherwise, we fall back to the "regular" insertion into the flat buffer - // (since it is not full anymore after removing the previous vector stored under the label). + // (since it is not full anymore after removing the previous vector stored under the + // label). } this->flatIndexGuard.lock(); idType new_flat_id = this->frontendIndex->indexSize(); @@ -814,8 +826,8 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l old_job->id = this->setAndSaveInvalidJob(old_job); this->labelToInsertJobs.erase(label); ret = 0; - // We are going to update the internal id that currently holds the vector associated with - // the given label. + // We are going to update the internal id that currently holds the vector associated + // with the given label. new_flat_id = dynamic_cast *>(this->frontendIndex) ->getIdOfLabel(label); @@ -827,10 +839,11 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l AsyncJob *new_insert_job = new (this->allocator) HNSWInsertJob(this->allocator, label, new_flat_id, executeInsertJobWrapper, this); - // Save a pointer to the job, so that if the vector is overwritten, we'll have an indication. + // Save a pointer to the job, so that if the vector is overwritten, we'll have an + // indication. if (this->labelToInsertJobs.find(label) != this->labelToInsertJobs.end()) { - // There's already a pending insert job for this label, add another one (without overwrite, - // only possible in multi index) + // There's already a pending insert job for this label, add another one (without + // overwrite, only possible in multi index) assert(this->backendIndex->isMultiValue()); this->labelToInsertJobs.at(label).push_back((HNSWInsertJob *)new_insert_job); } else { @@ -851,8 +864,8 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l // Apply ready swap jobs if number of deleted vectors reached the threshold (under exclusive // lock of the main index guard). // If swapJobs size is equal or larger than a threshold, go over the swap jobs and execute a - // batch of jobs for which all of its pending repair jobs were executed (otherwise finish and - // return). + // batch of jobs for which all of its pending repair jobs were executed (otherwise finish + // and return). if (readySwapJobs >= this->pendingSwapJobsThreshold) { this->executeReadySwapJobs(this->pendingSwapJobsThreshold); } @@ -869,7 +882,8 @@ int TieredHNSWIndex::deleteVector(labelType label) { if (this->frontendIndex->isLabelExists(label)) { this->flatIndexGuard.unlock_shared(); this->flatIndexGuard.lock(); - // Check again if the label exists, as it may have been removed while we released the lock. + // Check again if the label exists, as it may have been removed while we released the + // lock. if (this->frontendIndex->isLabelExists(label)) { // Invalidate the pending insert job(s) into HNSW associated with this label auto &insert_jobs = this->labelToInsertJobs.at(label); @@ -879,10 +893,10 @@ int TieredHNSWIndex::deleteVector(labelType label) { num_deleted_vectors += insert_jobs.size(); // Remove the pending insert job(s) from the labelToInsertJobs mapping. this->labelToInsertJobs.erase(label); - // Go over the every id that corresponds the label and remove it from the flat buffer. - // Every delete may cause a swap of the deleted id with the last id, and we return a - // mapping from id to the original id that resides in this id after the deletion(s) (see - // an example in this function implementation in MULTI index). + // Go over the every id that corresponds the label and remove it from the flat + // buffer. Every delete may cause a swap of the deleted id with the last id, and we + // return a mapping from id to the original id that resides in this id after the + // deletion(s) (see an example in this function implementation in MULTI index). auto updated_ids = this->frontendIndex->deleteVectorAndGetUpdatedIds(label); for (auto &it : updated_ids) { idType prev_id = it.second.first; @@ -895,11 +909,11 @@ int TieredHNSWIndex::deleteVector(labelType label) { this->flatIndexGuard.unlock_shared(); } - // Next, check if there vector(s) stored under the given label in HNSW and delete them as well. - // Note that we may remove the same vector that has been removed from the flat index, if it was - // being ingested at that time. - // writeMode is not protected since it is assumed to be called only from the "main thread" - // (that is the thread that is exclusively calling add/delete vector). + // Next, check if there vector(s) stored under the given label in HNSW and delete them as + // well. Note that we may remove the same vector that has been removed from the flat index, + // if it was being ingested at that time. writeMode is not protected since it is assumed to + // be called only from the "main thread" (that is the thread that is exclusively calling + // add/delete vector). if (this->getWriteMode() == VecSim_WriteAsync) { num_deleted_vectors += this->deleteLabelFromHNSW(label); // Apply ready swap jobs if number of deleted vectors reached the threshold @@ -919,20 +933,20 @@ int TieredHNSWIndex::deleteVector(labelType label) { /** * Move a label across every place the tiered index records one. Unlike add/delete this needs no - * jobs and no vector data movement - the swap/repair machinery is keyed purely on internal ids, so - * it is unaffected. + * jobs and no vector data movement - the swap/repair machinery is keyed purely on internal ids, + * so it is unaffected. * * A label can legitimately live in *both* tiers at once: `executeInsertJob` inserts into HNSW * before removing the vector from the flat buffer, so all four updates below are independently * guarded by their own existence check rather than treated as mutually exclusive. * - * Locking: `flatIndexGuard` exclusively (it guards the flat buffer's lookups, `labelToInsertJobs` - * and - per the note in `executeInsertJob` - the job fields themselves), then the main index guard - * exclusively. That is the flat-then-main order used by `insertVectorToHNSW` and `topKQueryImp`, - * so it cannot deadlock against them; repair and swap jobs only ever take the main guard. - * The main guard must be *exclusive* rather than shared: query threads read - * `getExternalLabel` under a shared main guard, and `ElementMetaData` is byte-packed, so its - * `label` field can be unaligned and its store is not atomic. + * Locking: `flatIndexGuard` exclusively (it guards the flat buffer's lookups, + * `labelToInsertJobs` and - per the note in `executeInsertJob` - the job fields themselves), + * then the main index guard exclusively. That is the flat-then-main order used by + * `insertVectorToHNSW` and `topKQueryImp`, so it cannot deadlock against them; repair and swap + * jobs only ever take the main guard. The main guard must be *exclusive* rather than shared: + * query threads read `getExternalLabel` under a shared main guard, and `ElementMetaData` is + * byte-packed, so its `label` field can be unaligned and its store is not atomic. */ template VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType old_label, @@ -946,9 +960,9 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o auto *hnsw_index = this->getHNSWIndex(); // A label can live in any subset of its three homes, so both questions are asked of all of - // them: it is present if any home holds it, and the target is free only if none does. Reject if - // the target is taken anywhere, so that a partially applied move is impossible - a free target - // in all three homes means none of the updates below can collide. + // them: it is present if any home holds it, and the target is free only if none does. + // Reject if the target is taken anywhere, so that a partially applied move is impossible - + // a free target in all three homes means none of the updates below can collide. const bool source_exists = this->frontendIndex->isLabelExists(old_label) || this->labelToInsertJobs.find(old_label) != this->labelToInsertJobs.end() || @@ -958,10 +972,11 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o this->labelToInsertJobs.find(new_label) != this->labelToInsertJobs.end() || hnsw_index->isLabelExists(new_label); - // Each home is asked to move the label only once it reported holding it, and the checks above - // ruled out every other rejection - the label is present, the target is free everywhere, and - // the labels differ - so while both guards are held a home that is asked can only answer OK. - // The asserts below state that; a refusal would mean the state changed underneath us. + // Each home is asked to move the label only once it reported holding it, and the checks + // above ruled out every other rejection - the label is present, the target is free + // everywhere, and the labels differ - so while both guards are held a home that is asked + // can only answer OK. The asserts below state that; a refusal would mean the state changed + // underneath us. if (source_exists && !target_taken) { if (this->frontendIndex->isLabelExists(old_label)) { const VecSimRelabelCode flat_ret = @@ -973,10 +988,10 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o UNUSED(flat_ret); } - // Re-key the pending insert jobs *and* rewrite each job's own copy of the label. Both must - // move together: `executeInsertJob` indexes into HNSW under `job->label` and then looks the - // job up with `labelToInsertJobs.at(job->label)`, so a half-applied move either indexes the - // vector under the stale label or throws out of the worker thread. + // Re-key the pending insert jobs *and* rewrite each job's own copy of the label. Both + // must move together: `executeInsertJob` indexes into HNSW under `job->label` and then + // looks the job up with `labelToInsertJobs.at(job->label)`, so a half-applied move + // either indexes the vector under the stale label or throws out of the worker thread. auto jobs_it = this->labelToInsertJobs.find(old_label); if (jobs_it != this->labelToInsertJobs.end()) { auto jobs = std::move(jobs_it->second); @@ -1002,7 +1017,8 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o this->flatIndexGuard.unlock(); // An absent source outranks an occupied target: a caller that resolves `NewLabelTaken` by - // freeing the target would otherwise drop an unrelated vector for a move with nothing to move. + // freeing the target would otherwise drop an unrelated vector for a move with nothing to + // move. if (!source_exists) { return VecSimRelabel_OldLabelMissing; } @@ -1015,25 +1031,30 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o // `getDistanceFrom` returns the minimum distance between the given blob and the vector with the // given label. If the label doesn't exist, the distance will be NaN. // Therefore, it's better to just call `getDistanceFrom` on both indexes and return the minimum -// instead of checking if the label exists in each index. We first try to get the distance from the -// flat buffer, as vectors in the buffer might move to the Main while we're "between" the locks. -// Behavior for single (regular) index: +// instead of checking if the label exists in each index. We first try to get the distance from +// the flat buffer, as vectors in the buffer might move to the Main while we're "between" the +// locks. Behavior for single (regular) index: // 1. label doesn't exist in both indexes - return NaN -// 2. label exists in one of the indexes only - return the distance from that index (which is valid) -// 3. label exists in both indexes - return the value from the flat buffer (which is valid and equal +// 2. label exists in one of the indexes only - return the distance from that index (which is +// valid) +// 3. label exists in both indexes - return the value from the flat buffer (which is valid and +// equal // to the value from the Main index), saving us from locking the Main index. // Behavior for multi index: // 1. label doesn't exist in both indexes - return NaN -// 2. label exists in one of the indexes only - return the distance from that index (which is valid) -// 3. label exists in both indexes - we may have some of the vectors with the same label in the flat +// 2. label exists in one of the indexes only - return the distance from that index (which is +// valid) +// 3. label exists in both indexes - we may have some of the vectors with the same label in the +// flat // buffer only and some in the Main index only (and maybe temporal duplications). // So, we get the distance from both indexes and return the minimum. -// IMPORTANT: this should be called when the *tiered index locks are locked for shared ownership*, -// along with HNSW index data guard lock. That is since the internal getDistanceFrom calls access -// the indexes' data, and it is not safe to run insert/delete operation in parallel. Also, we avoid -// acquiring the locks internally, since this is usually called for every vector individually, and -// the overhead of acquiring and releasing the locks is significant in that case. +// IMPORTANT: this should be called when the *tiered index locks are locked for shared +// ownership*, along with HNSW index data guard lock. That is since the internal getDistanceFrom +// calls access the indexes' data, and it is not safe to run insert/delete operation in +// parallel. Also, we avoid acquiring the locks internally, since this is usually called for +// every vector individually, and the overhead of acquiring and releasing the locks is +// significant in that case. template double TieredHNSWIndex::getDistanceFrom_Unsafe(labelType label, const void *blob) const { @@ -1041,8 +1062,8 @@ double TieredHNSWIndex::getDistanceFrom_Unsafe(labelType lab // If the label doesn't exist, the distance will be NaN. auto flat_dist = this->frontendIndex->getDistanceFrom_Unsafe(label, blob); - // Optimization. TODO: consider having different implementations for single and multi indexes, - // to avoid checking the index type on every query. + // Optimization. TODO: consider having different implementations for single and multi + // indexes, to avoid checking the index type on every query. if (!this->backendIndex->isMultiValue() && !std::isnan(flat_dist)) { // If the index is single value, and we got a valid distance from the flat buffer, // we can return the distance without querying the Main index. @@ -1074,8 +1095,8 @@ TieredHNSWIndex::TieredHNSW_BatchIterator::TieredHNSW_BatchI // Tiered batch iterator doesn't hold its own copy of the query vector. // Instead, each internal batch iterators (flat_iterator and hnsw_iterator) create their own // copies: flat_iterator copy is created during TieredHNSW_BatchIterator construction When - // TieredHNSW_BatchIterator::getNextResults() is called and hnsw_iterator is not initialized, it - // retrieves the blob from flat_iterator + // TieredHNSW_BatchIterator::getNextResults() is called and hnsw_iterator is not + // initialized, it retrieves the blob from flat_iterator : VecSimBatchIterator(nullptr, queryParams ? queryParams->timeoutCtx : nullptr, std::move(allocator)), index(index), flat_results(this->allocator), hnsw_results(this->allocator), @@ -1114,8 +1135,8 @@ VecSimQueryReply *TieredHNSWIndex::TieredHNSW_BatchIterator: auto hnsw_code = VecSim_QueryReply_OK; if (this->hnsw_iterator == UNINITIALIZED) { - // First call to getNextResults. The call to the BF iterator will include calculating all - // the distances and access the BF index. We take the lock on this call. + // First call to getNextResults. The call to the BF iterator will include calculating + // all the distances and access the BF index. We take the lock on this call. this->index->flatIndexGuard.lock_shared(); auto cur_flat_results = this->flat_iterator->getNextResults(n_res, BY_SCORE_THEN_ID); this->index->flatIndexGuard.unlock_shared(); @@ -1125,8 +1146,8 @@ VecSimQueryReply *TieredHNSWIndex::TieredHNSW_BatchIterator: } this->flat_results.swap(cur_flat_results->results); VecSimQueryReply_Free(cur_flat_results); - // We also take the lock on the main index on the first call to getNextResults, and we hold - // it until the iterator is depleted or freed. + // We also take the lock on the main index on the first call to getNextResults, and we + // hold it until the iterator is depleted or freed. this->index->mainIndexGuard.lock_shared(); this->hnsw_iterator = this->index->backendIndex->newBatchIterator( this->flat_iterator->getQueryBlob(), queryParams); @@ -1148,10 +1169,10 @@ VecSimQueryReply *TieredHNSWIndex::TieredHNSW_BatchIterator: VecSimQueryReply_Free(tail); if (!isMulti) { - // On single-value indexes, duplicates will never appear in the hnsw results before - // they appear in the flat results (at the same time or later if the approximation - // misses) so we don't need to try and filter the flat results (and recheck - // conditions). + // On single-value indexes, duplicates will never appear in the hnsw results + // before they appear in the flat results (at the same time or later if the + // approximation misses) so we don't need to try and filter the flat results + // (and recheck conditions). break; } else { // On multi-value indexes, the flat results may contain results that are already @@ -1165,9 +1186,9 @@ VecSimQueryReply *TieredHNSWIndex::TieredHNSW_BatchIterator: auto tail = this->hnsw_iterator->getNextResults(n_res - this->hnsw_results.size(), BY_SCORE_THEN_ID); hnsw_code = tail->code; // Set the hnsw_results code to the last `getNextResults` code. - // New batch may contain better results than the previous batch, so we need to merge. - // We don't expect duplications (hence the ), as the iterator guarantees that - // no result is returned twice. + // New batch may contain better results than the previous batch, so we need to + // merge. We don't expect duplications (hence the ), as the iterator + // guarantees that no result is returned twice. VecSimQueryResultContainer cur_hnsw_results(this->allocator); merge_results(cur_hnsw_results, this->hnsw_results, tail->results, n_res); VecSimQueryReply_Free(tail); @@ -1244,13 +1265,15 @@ TieredHNSWIndex::TieredHNSW_BatchIterator::compute_current_b auto [from_hnsw, from_flat] = p; if (!isMultiValue) { - // If we're on a single-value index, update the set of results returned from the FLAT index - // before popping them, to prevent them to be returned from the HNSW index in later batches. + // If we're on a single-value index, update the set of results returned from the FLAT + // index before popping them, to prevent them to be returned from the HNSW index in + // later batches. for (size_t i = 0; i < from_flat; ++i) { this->returned_results_set.insert(this->flat_results[i].id); } } else { - // If we're on a multi-value index, update the set of results returned (from `batch_res`) + // If we're on a multi-value index, update the set of results returned (from + // `batch_res`) for (size_t i = 0; i < batch_res->results.size(); ++i) { this->returned_results_set.insert(batch_res->results[i].id); } diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index a9836e3ab..4406f5e48 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -59,13 +59,13 @@ class VecSimTieredIndex : public VecSimIndexInterface { SharedMutexLockable flatIndexLockable{flatIndexGuard}; SharedMutexLockable mainIndexLockable{mainIndexGuard}; - // Locking behavior for topKQuery/rangeQuery: real lock by default, overridden to a no-op - // by backends (e.g. SVS) that handle their own concurrency internally. - virtual ScopedLocks lockMainIndexForQuery() const { return ScopedLocks(mainIndexLockable); } + // Locking behavior for topKQuery/rangeQuery + virtual ScopedLocks lockMainIndexForQuery() const; - // Locking behavior for indexSize/indexCapacity, which each backend defines to match - // whatever it actually needs to guard those counters with -- the two differ per backend. + // Locking behavior for indexSize() virtual ScopedLocks lockIndexForSize() const = 0; + + // Locking behavior for indexCapacity() virtual ScopedLocks lockIndexForCapacity() const = 0; void lockMainIndexGuard() const { From bb119407d811a48d61b4eef66ab29327d373a9d4 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Mon, 14 Sep 2026 07:16:18 -0700 Subject: [PATCH 43/64] fixup --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 4 +++- src/VecSim/vec_sim_tiered_index.h | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index 0b3f3d968..8322e870d 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -166,7 +166,9 @@ class TieredHNSWIndex : public VecSimTieredIndex { }; HnswDataGuardLockable hnswDataGuardLockable{this->getHNSWIndex()}; - ScopedLocks lockMainIndexForQuery() { return ScopedLocks(mainIndexLockable); } + ScopedLocks lockMainIndexForQuery() const override { + return ScopedLocks(this->mainIndexLockable); + } ScopedLocks lockIndexForSize() const override { return ScopedLocks(this->flatIndexLockable, hnswDataGuardLockable); diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 4406f5e48..628cb6a65 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -60,7 +60,7 @@ class VecSimTieredIndex : public VecSimIndexInterface { SharedMutexLockable mainIndexLockable{mainIndexGuard}; // Locking behavior for topKQuery/rangeQuery - virtual ScopedLocks lockMainIndexForQuery() const; + virtual ScopedLocks lockMainIndexForQuery() const = 0; // Locking behavior for indexSize() virtual ScopedLocks lockIndexForSize() const = 0; From c0d3a9631f71bed106259407890b2052e992e608 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Mon, 14 Sep 2026 07:23:38 -0700 Subject: [PATCH 44/64] fix comment formatting --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 277 +++++++++++------------ 1 file changed, 129 insertions(+), 148 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index 8322e870d..e30576169 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -191,15 +191,16 @@ class TieredHNSWIndex : public VecSimTieredIndex { VecSimBatchIterator *flat_iterator; VecSimBatchIterator *hnsw_iterator; - // On single value indices, this set holds the IDs of the results that were returned - // from the flat buffer. On multi value indices, this set holds the IDs of all the - // results that were returned. The difference between the two cases is that on multi - // value indices, the same ID can appear in both indexes and results with different - // scores, and therefore we can't tell in advance when we expect a possibility of a - // duplicate. On single value indices, a duplicate may appear at the same batch (and we - // will handle it when merging the results) Or it may appear in a different batches, - // first from the flat buffer and then from the HNSW, in the cases where a better result - // if found later in HNSW because of the approximate nature of the algorithm. + // On single value indices, this set holds the IDs of the results that were returned from + // the flat buffer. + // On multi value indices, this set holds the IDs of all the results that were returned. + // The difference between the two cases is that on multi value indices, the same ID can + // appear in both indexes and results with different scores, and therefore we can't tell in + // advance when we expect a possibility of a duplicate. + // On single value indices, a duplicate may appear at the same batch (and we will handle it + // when merging the results) Or it may appear in a different batches, first from the flat + // buffer and then from the HNSW, in the cases where a better result if found later in HNSW + // because of the approximate nature of the algorithm. vecsim_stl::unordered_set returned_results_set; private: @@ -242,8 +243,8 @@ class TieredHNSWIndex : public VecSimTieredIndex { return this->getHNSWIndex()->getNumMarkedDeleted(); } double getDistanceFrom_Unsafe(labelType label, const void *blob) const override; - // Do nothing here, each tier (flat buffer and HNSW) should increase capacity for itself - // when needed. + // Do nothing here, each tier (flat buffer and HNSW) should increase capacity for itself when + // needed. VecSimIndexDebugInfo debugInfo() const override; VecSimIndexBasicInfo basicInfo() const override; VecSimIndexStatsInfo statisticInfo() const override; @@ -325,16 +326,16 @@ void TieredHNSWIndex::fixJobsAfterSwap( idToRepairJobs.insert({deleted_id, idToRepairJobs.at(prev_last_id)}); idToRepairJobs.erase(prev_last_id); } - // Update the swap jobs if the last id also needs a swap, otherwise just collect to deleted - // id to be removed from the swap jobs. + // Update the swap jobs if the last id also needs a swap, otherwise just collect to deleted id + // to be removed from the swap jobs. if (prev_last_id != deleted_id && idToSwapJob.find(prev_last_id) != idToSwapJob.end() && std::find(idsToRemove.begin(), idsToRemove.end(), prev_last_id) == idsToRemove.end()) { - // Update the curr_last_id pending swap job id after the removal that renamed - // curr_last_id with the deleted id. + // Update the curr_last_id pending swap job id after the removal that renamed curr_last_id + // with the deleted id. idsToRemove.push_back(prev_last_id); idToSwapJob.at(prev_last_id)->deleted_id = deleted_id; - // If id was deleted in-place and there is no swap job for it, this will create a new - // entry in idToSwapJob for the swapped id, otherwise it will update the existing entry. + // If id was deleted in-place and there is no swap job for it, this will create a new entry + // in idToSwapJob for the swapped id, otherwise it will update the existing entry. idToSwapJob[deleted_id] = idToSwapJob.at(prev_last_id); } else { idsToRemove.push_back(deleted_id); @@ -343,8 +344,7 @@ void TieredHNSWIndex::fixJobsAfterSwap( template void TieredHNSWIndex::invalidateRepairJobs(idType deleted_id) { - // Invalidate repair jobs for the disposed id (if exist), and update the associated swap - // jobs. + // Invalidate repair jobs for the disposed id (if exist), and update the associated swap jobs. if (idToRepairJobs.find(deleted_id) == idToRepairJobs.end()) { return; } @@ -459,9 +459,9 @@ int TieredHNSWIndex::deleteLabelFromHNSW(labelType label) { this->idToRepairJobsGuard.unlock(); if (incomingEdges.size() == 0) { - // No repair job will ever run for this element, so this is already the point at - // which it can be taken out of the graph (outside the repair jobs guard, as - // isolating takes the per-element links locks). + // No repair job will ever run for this element, so this is already the point at which + // it can be taken out of the graph (outside the repair jobs guard, as isolating takes + // the per-element links locks). this->isolateRepairedElement(id); } @@ -501,8 +501,8 @@ void TieredHNSWIndex::insertVectorToHNSW( const void *processed_storage_blob = processed_blobs.getStorageBlob(); const void *processed_for_index = processed_blobs.getQueryBlob(); - // Acquire the index data lock, so we know what is the exact index size at this time. - // Acquire the main r/w lock before to avoid deadlocks. + // Acquire the index data lock, so we know what is the exact index size at this time. Acquire + // the main r/w lock before to avoid deadlocks. this->mainIndexGuard.lock_shared(); hnsw_index->lockIndexDataGuard(); // Check if resizing is needed for HNSW index (requires write lock). @@ -513,11 +513,11 @@ void TieredHNSWIndex::insertVectorToHNSW( this->lockMainIndexGuard(); hnsw_index->lockIndexDataGuard(); - // Hold the index data lock while we store the new element. If the new node's max level - // is higher than the current one, hold the lock through the entire insertion to ensure - // that graph scans will not occur, as they will try access the entry point's neighbors. - // If an index resize is still needed, `storeNewElement` will perform it. This is OK - // since we hold the main index lock for exclusive access. + // Hold the index data lock while we store the new element. If the new node's max level is + // higher than the current one, hold the lock through the entire insertion to ensure that + // graph scans will not occur, as they will try access the entry point's neighbors. + // If an index resize is still needed, `storeNewElement` will perform it. This is OK since + // we hold the main index lock for exclusive access. auto state = hnsw_index->storeNewElement(label, processed_storage_blob); if constexpr (releaseFlatGuard) { this->flatIndexGuard.unlock_shared(); @@ -526,13 +526,11 @@ void TieredHNSWIndex::insertVectorToHNSW( // If we're still holding the index data guard, we cannot take the main index lock for // shared ownership as it may cause deadlocks, and we also cannot release the main index // lock between, since we cannot allow swap jobs to happen, as they will make the - // saved state invalid. Hence, we insert the vector with the current exclusive lock - // held. + // saved state invalid. Hence, we insert the vector with the current exclusive lock held. if (state.elementMaxLevel <= state.currMaxLevel) { hnsw_index->unlockIndexDataGuard(); } - // Take the vector from the flat buffer and insert it to HNSW (overwrite should not - // occur). + // Take the vector from the flat buffer and insert it to HNSW (overwrite should not occur). hnsw_index->indexVector(processed_for_index, label, state); if (state.elementMaxLevel > state.currMaxLevel) { hnsw_index->unlockIndexDataGuard(); @@ -540,11 +538,11 @@ void TieredHNSWIndex::insertVectorToHNSW( this->unlockMainIndexGuard(); } else { // Do the same as above except for changing the capacity, but with *shared* lock held: - // Hold the index data lock while we store the new element. If the new node's max level - // is higher than the current one, hold the lock through the entire insertion to ensure - // that graph scans will not occur, as they will try access the entry point's neighbors. - // At this point we are certain that the index has enough capacity for the new element, - // and this call will not resize the index. + // Hold the index data lock while we store the new element. If the new node's max level is + // higher than the current one, hold the lock through the entire insertion to ensure that + // graph scans will not occur, as they will try access the entry point's neighbors. + // At this point we are certain that the index has enough capacity for the new element, and + // this call will not resize the index. auto state = hnsw_index->storeNewElement(label, processed_storage_blob); if constexpr (releaseFlatGuard) { this->flatIndexGuard.unlock_shared(); @@ -553,8 +551,7 @@ void TieredHNSWIndex::insertVectorToHNSW( if (state.elementMaxLevel <= state.currMaxLevel) { hnsw_index->unlockIndexDataGuard(); } - // Take the vector from the flat buffer and insert it to HNSW (overwrite should not - // occur). + // Take the vector from the flat buffer and insert it to HNSW (overwrite should not occur). hnsw_index->indexVector(processed_for_index, label, state); if (state.elementMaxLevel > state.currMaxLevel) { hnsw_index->unlockIndexDataGuard(); @@ -600,8 +597,7 @@ int TieredHNSWIndex::deleteLabelFromHNSWInplace(labelType la /******************** Job's callbacks **********************************/ template void TieredHNSWIndex::executeInsertJob(HNSWInsertJob *job) { - // Note that accessing the job fields should occur with flat index guard held (here and - // later). + // Note that accessing the job fields should occur with flat index guard held (here and later). this->flatIndexGuard.lock_shared(); if (!job->isValid) { this->flatIndexGuard.unlock_shared(); @@ -640,22 +636,20 @@ void TieredHNSWIndex::executeInsertJob(HNSWInsertJob *job) { if (labelToInsertJobs.at(job->label).empty()) { labelToInsertJobs.erase(job->label); } - // Remove the vector from the flat buffer. This may cause the last vector id to swap - // with the deleted id. Hold the label for the last id, so we can later on update its + // Remove the vector from the flat buffer. This may cause the last vector id to swap with + // the deleted id. Hold the label for the last id, so we can later on update its // corresponding job id. Note that after calling deleteVectorById, the last id's label // shouldn't be available, since it is removed from the lookup. labelType last_vec_label = this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); int deleted = this->frontendIndex->deleteVectorById(job->label, job->id); if (deleted && job->id != this->frontendIndex->indexSize()) { - // If the vector removal caused a swap with the last id, update the relevant insert - // job. + // If the vector removal caused a swap with the last id, update the relevant insert job. this->updateInsertJobInternalId(this->frontendIndex->indexSize(), job->id, last_vec_label); } } else { - // Remove the current job from the invalid jobs' lookup, as we are about to delete it - // now. + // Remove the current job from the invalid jobs' lookup, as we are about to delete it now. this->invalidJobsLookupGuard.lock(); this->invalidJobs.erase(job->id); this->invalidJobsLookupGuard.unlock(); @@ -677,10 +671,10 @@ void TieredHNSWIndex::executeRepairJob(HNSWRepairJob *job) { } HNSWIndex *hnsw_index = this->getHNSWIndex(); - // Remove this job pointer from the repair jobs lookup BEFORE it has been executed. Had we - // done it after executing the repair job, we might have see that there is a pending repair - // job for this node id upon deleting another neighbor of this node, and we may avoid - // creating another repair job even though *it has already been executed*. + // Remove this job pointer from the repair jobs lookup BEFORE it has been executed. Had we done + // it after executing the repair job, we might have see that there is a pending repair job for + // this node id upon deleting another neighbor of this node, and we may avoid creating another + // repair job even though *it has already been executed*. this->idToRepairJobsGuard.lock(); auto &repair_jobs = this->idToRepairJobs.at(job->node_id); assert(repair_jobs.size() > 0); @@ -713,10 +707,9 @@ void TieredHNSWIndex::executeRepairJob(HNSWRepairJob *job) { } this->idToRepairJobsGuard.unlock(); - // These deleted elements have no pending repair job left, so nothing points to them - // anymore. Take them out of the graph entirely, leaving no edge in or out for the swap job - // to deal with. Done outside the repair jobs guard, as isolating takes the per-element - // links locks. + // These deleted elements have no pending repair job left, so nothing points to them anymore. + // Take them out of the graph entirely, leaving no edge in or out for the swap job to deal with. + // Done outside the repair jobs guard, as isolating takes the per-element links locks. for (idType deleted_id : fully_repaired_ids) { this->isolateRepairedElement(deleted_id); } @@ -770,9 +763,9 @@ TieredHNSWIndex::~TieredHNSWIndex() { // In the tiered index, we assume that the blobs are processed by the flat buffer // before being transferred to the HNSW index. -// When inserting vectors directly into the HNSW index—such as in VecSim_WriteInPlace mode— or -// when the flat buffer is full, we must manually preprocess the blob according to the -// **frontend** index parameters. +// When inserting vectors directly into the HNSW index—such as in VecSim_WriteInPlace mode— or when +// the flat buffer is full, we must manually preprocess the blob according to the **frontend** index +// parameters. template int TieredHNSWIndex::addVector(const void *blob, labelType label) { int ret = 1; @@ -780,8 +773,7 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l // writeMode is not protected since it is assumed to be called only from the "main thread" // (that is the thread that is exclusively calling add/delete vector). if (this->getWriteMode() == VecSim_WriteInPlace) { - // First, check if we need to overwrite the vector in-place for single (from both - // indexes). + // First, check if we need to overwrite the vector in-place for single (from both indexes). if (!this->backendIndex->isMultiValue()) { ret -= this->deleteVector(label); } @@ -801,15 +793,16 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l if (this->frontendIndex->indexSize() >= this->flatBufferLimit) { // Handle overwrite situation. if (!this->backendIndex->isMultiValue()) { - // This will do nothing (and return 0) if this label doesn't exist. Otherwise, it - // may remove vector from the flat buffer and/or the HNSW index. + // This will do nothing (and return 0) if this label doesn't exist. Otherwise, it may + // remove vector from the flat buffer and/or the HNSW index. ret -= this->deleteVector(label); } if (this->frontendIndex->indexSize() >= this->flatBufferLimit) { - // We didn't remove a vector from flat buffer due to overwrite, insert the new - // vector directly to HNSW. Since flat buffer guard was not held, no need to release - // it internally. Use the frontend parameters to manually prepare the blob for its - // transfer to the HNSW index. + // We didn't remove a vector from flat buffer due to overwrite, insert the new vector + // directly to HNSW. Since flat buffer guard was not held, no need to release it + // internally. + // Use the frontend parameters to manually prepare the blob for its transfer to the HNSW + // index. auto storage_blob = this->frontendIndex->preprocessForStorage(blob); this->insertVectorToHNSW(hnsw_index, label, storage_blob.get()); // Track direct insertion to HNSW (flat buffer was full) @@ -817,8 +810,7 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l return ret; } // Otherwise, we fall back to the "regular" insertion into the flat buffer - // (since it is not full anymore after removing the previous vector stored under the - // label). + // (since it is not full anymore after removing the previous vector stored under the label). } this->flatIndexGuard.lock(); idType new_flat_id = this->frontendIndex->indexSize(); @@ -828,8 +820,8 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l old_job->id = this->setAndSaveInvalidJob(old_job); this->labelToInsertJobs.erase(label); ret = 0; - // We are going to update the internal id that currently holds the vector associated - // with the given label. + // We are going to update the internal id that currently holds the vector associated with + // the given label. new_flat_id = dynamic_cast *>(this->frontendIndex) ->getIdOfLabel(label); @@ -841,11 +833,10 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l AsyncJob *new_insert_job = new (this->allocator) HNSWInsertJob(this->allocator, label, new_flat_id, executeInsertJobWrapper, this); - // Save a pointer to the job, so that if the vector is overwritten, we'll have an - // indication. + // Save a pointer to the job, so that if the vector is overwritten, we'll have an indication. if (this->labelToInsertJobs.find(label) != this->labelToInsertJobs.end()) { - // There's already a pending insert job for this label, add another one (without - // overwrite, only possible in multi index) + // There's already a pending insert job for this label, add another one (without overwrite, + // only possible in multi index) assert(this->backendIndex->isMultiValue()); this->labelToInsertJobs.at(label).push_back((HNSWInsertJob *)new_insert_job); } else { @@ -866,8 +857,8 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l // Apply ready swap jobs if number of deleted vectors reached the threshold (under exclusive // lock of the main index guard). // If swapJobs size is equal or larger than a threshold, go over the swap jobs and execute a - // batch of jobs for which all of its pending repair jobs were executed (otherwise finish - // and return). + // batch of jobs for which all of its pending repair jobs were executed (otherwise finish and + // return). if (readySwapJobs >= this->pendingSwapJobsThreshold) { this->executeReadySwapJobs(this->pendingSwapJobsThreshold); } @@ -884,8 +875,7 @@ int TieredHNSWIndex::deleteVector(labelType label) { if (this->frontendIndex->isLabelExists(label)) { this->flatIndexGuard.unlock_shared(); this->flatIndexGuard.lock(); - // Check again if the label exists, as it may have been removed while we released the - // lock. + // Check again if the label exists, as it may have been removed while we released the lock. if (this->frontendIndex->isLabelExists(label)) { // Invalidate the pending insert job(s) into HNSW associated with this label auto &insert_jobs = this->labelToInsertJobs.at(label); @@ -895,10 +885,10 @@ int TieredHNSWIndex::deleteVector(labelType label) { num_deleted_vectors += insert_jobs.size(); // Remove the pending insert job(s) from the labelToInsertJobs mapping. this->labelToInsertJobs.erase(label); - // Go over the every id that corresponds the label and remove it from the flat - // buffer. Every delete may cause a swap of the deleted id with the last id, and we - // return a mapping from id to the original id that resides in this id after the - // deletion(s) (see an example in this function implementation in MULTI index). + // Go over the every id that corresponds the label and remove it from the flat buffer. + // Every delete may cause a swap of the deleted id with the last id, and we return a + // mapping from id to the original id that resides in this id after the deletion(s) (see + // an example in this function implementation in MULTI index). auto updated_ids = this->frontendIndex->deleteVectorAndGetUpdatedIds(label); for (auto &it : updated_ids) { idType prev_id = it.second.first; @@ -911,11 +901,11 @@ int TieredHNSWIndex::deleteVector(labelType label) { this->flatIndexGuard.unlock_shared(); } - // Next, check if there vector(s) stored under the given label in HNSW and delete them as - // well. Note that we may remove the same vector that has been removed from the flat index, - // if it was being ingested at that time. writeMode is not protected since it is assumed to - // be called only from the "main thread" (that is the thread that is exclusively calling - // add/delete vector). + // Next, check if there vector(s) stored under the given label in HNSW and delete them as well. + // Note that we may remove the same vector that has been removed from the flat index, if it was + // being ingested at that time. + // writeMode is not protected since it is assumed to be called only from the "main thread" + // (that is the thread that is exclusively calling add/delete vector). if (this->getWriteMode() == VecSim_WriteAsync) { num_deleted_vectors += this->deleteLabelFromHNSW(label); // Apply ready swap jobs if number of deleted vectors reached the threshold @@ -935,20 +925,20 @@ int TieredHNSWIndex::deleteVector(labelType label) { /** * Move a label across every place the tiered index records one. Unlike add/delete this needs no - * jobs and no vector data movement - the swap/repair machinery is keyed purely on internal ids, - * so it is unaffected. + * jobs and no vector data movement - the swap/repair machinery is keyed purely on internal ids, so + * it is unaffected. * * A label can legitimately live in *both* tiers at once: `executeInsertJob` inserts into HNSW * before removing the vector from the flat buffer, so all four updates below are independently * guarded by their own existence check rather than treated as mutually exclusive. * - * Locking: `flatIndexGuard` exclusively (it guards the flat buffer's lookups, - * `labelToInsertJobs` and - per the note in `executeInsertJob` - the job fields themselves), - * then the main index guard exclusively. That is the flat-then-main order used by - * `insertVectorToHNSW` and `topKQueryImp`, so it cannot deadlock against them; repair and swap - * jobs only ever take the main guard. The main guard must be *exclusive* rather than shared: - * query threads read `getExternalLabel` under a shared main guard, and `ElementMetaData` is - * byte-packed, so its `label` field can be unaligned and its store is not atomic. + * Locking: `flatIndexGuard` exclusively (it guards the flat buffer's lookups, `labelToInsertJobs` + * and - per the note in `executeInsertJob` - the job fields themselves), then the main index guard + * exclusively. That is the flat-then-main order used by `insertVectorToHNSW` and `topKQueryImp`, + * so it cannot deadlock against them; repair and swap jobs only ever take the main guard. + * The main guard must be *exclusive* rather than shared: query threads read + * `getExternalLabel` under a shared main guard, and `ElementMetaData` is byte-packed, so its + * `label` field can be unaligned and its store is not atomic. */ template VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType old_label, @@ -962,9 +952,9 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o auto *hnsw_index = this->getHNSWIndex(); // A label can live in any subset of its three homes, so both questions are asked of all of - // them: it is present if any home holds it, and the target is free only if none does. - // Reject if the target is taken anywhere, so that a partially applied move is impossible - - // a free target in all three homes means none of the updates below can collide. + // them: it is present if any home holds it, and the target is free only if none does. Reject if + // the target is taken anywhere, so that a partially applied move is impossible - a free target + // in all three homes means none of the updates below can collide. const bool source_exists = this->frontendIndex->isLabelExists(old_label) || this->labelToInsertJobs.find(old_label) != this->labelToInsertJobs.end() || @@ -974,11 +964,10 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o this->labelToInsertJobs.find(new_label) != this->labelToInsertJobs.end() || hnsw_index->isLabelExists(new_label); - // Each home is asked to move the label only once it reported holding it, and the checks - // above ruled out every other rejection - the label is present, the target is free - // everywhere, and the labels differ - so while both guards are held a home that is asked - // can only answer OK. The asserts below state that; a refusal would mean the state changed - // underneath us. + // Each home is asked to move the label only once it reported holding it, and the checks above + // ruled out every other rejection - the label is present, the target is free everywhere, and + // the labels differ - so while both guards are held a home that is asked can only answer OK. + // The asserts below state that; a refusal would mean the state changed underneath us. if (source_exists && !target_taken) { if (this->frontendIndex->isLabelExists(old_label)) { const VecSimRelabelCode flat_ret = @@ -990,10 +979,10 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o UNUSED(flat_ret); } - // Re-key the pending insert jobs *and* rewrite each job's own copy of the label. Both - // must move together: `executeInsertJob` indexes into HNSW under `job->label` and then - // looks the job up with `labelToInsertJobs.at(job->label)`, so a half-applied move - // either indexes the vector under the stale label or throws out of the worker thread. + // Re-key the pending insert jobs *and* rewrite each job's own copy of the label. Both must + // move together: `executeInsertJob` indexes into HNSW under `job->label` and then looks the + // job up with `labelToInsertJobs.at(job->label)`, so a half-applied move either indexes the + // vector under the stale label or throws out of the worker thread. auto jobs_it = this->labelToInsertJobs.find(old_label); if (jobs_it != this->labelToInsertJobs.end()) { auto jobs = std::move(jobs_it->second); @@ -1019,8 +1008,7 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o this->flatIndexGuard.unlock(); // An absent source outranks an occupied target: a caller that resolves `NewLabelTaken` by - // freeing the target would otherwise drop an unrelated vector for a move with nothing to - // move. + // freeing the target would otherwise drop an unrelated vector for a move with nothing to move. if (!source_exists) { return VecSimRelabel_OldLabelMissing; } @@ -1033,30 +1021,25 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o // `getDistanceFrom` returns the minimum distance between the given blob and the vector with the // given label. If the label doesn't exist, the distance will be NaN. // Therefore, it's better to just call `getDistanceFrom` on both indexes and return the minimum -// instead of checking if the label exists in each index. We first try to get the distance from -// the flat buffer, as vectors in the buffer might move to the Main while we're "between" the -// locks. Behavior for single (regular) index: +// instead of checking if the label exists in each index. We first try to get the distance from the +// flat buffer, as vectors in the buffer might move to the Main while we're "between" the locks. +// Behavior for single (regular) index: // 1. label doesn't exist in both indexes - return NaN -// 2. label exists in one of the indexes only - return the distance from that index (which is -// valid) -// 3. label exists in both indexes - return the value from the flat buffer (which is valid and -// equal +// 2. label exists in one of the indexes only - return the distance from that index (which is valid) +// 3. label exists in both indexes - return the value from the flat buffer (which is valid and equal // to the value from the Main index), saving us from locking the Main index. // Behavior for multi index: // 1. label doesn't exist in both indexes - return NaN -// 2. label exists in one of the indexes only - return the distance from that index (which is -// valid) -// 3. label exists in both indexes - we may have some of the vectors with the same label in the -// flat +// 2. label exists in one of the indexes only - return the distance from that index (which is valid) +// 3. label exists in both indexes - we may have some of the vectors with the same label in the flat // buffer only and some in the Main index only (and maybe temporal duplications). // So, we get the distance from both indexes and return the minimum. -// IMPORTANT: this should be called when the *tiered index locks are locked for shared -// ownership*, along with HNSW index data guard lock. That is since the internal getDistanceFrom -// calls access the indexes' data, and it is not safe to run insert/delete operation in -// parallel. Also, we avoid acquiring the locks internally, since this is usually called for -// every vector individually, and the overhead of acquiring and releasing the locks is -// significant in that case. +// IMPORTANT: this should be called when the *tiered index locks are locked for shared ownership*, +// along with HNSW index data guard lock. That is since the internal getDistanceFrom calls access +// the indexes' data, and it is not safe to run insert/delete operation in parallel. Also, we avoid +// acquiring the locks internally, since this is usually called for every vector individually, and +// the overhead of acquiring and releasing the locks is significant in that case. template double TieredHNSWIndex::getDistanceFrom_Unsafe(labelType label, const void *blob) const { @@ -1064,8 +1047,8 @@ double TieredHNSWIndex::getDistanceFrom_Unsafe(labelType lab // If the label doesn't exist, the distance will be NaN. auto flat_dist = this->frontendIndex->getDistanceFrom_Unsafe(label, blob); - // Optimization. TODO: consider having different implementations for single and multi - // indexes, to avoid checking the index type on every query. + // Optimization. TODO: consider having different implementations for single and multi indexes, + // to avoid checking the index type on every query. if (!this->backendIndex->isMultiValue() && !std::isnan(flat_dist)) { // If the index is single value, and we got a valid distance from the flat buffer, // we can return the distance without querying the Main index. @@ -1097,8 +1080,8 @@ TieredHNSWIndex::TieredHNSW_BatchIterator::TieredHNSW_BatchI // Tiered batch iterator doesn't hold its own copy of the query vector. // Instead, each internal batch iterators (flat_iterator and hnsw_iterator) create their own // copies: flat_iterator copy is created during TieredHNSW_BatchIterator construction When - // TieredHNSW_BatchIterator::getNextResults() is called and hnsw_iterator is not - // initialized, it retrieves the blob from flat_iterator + // TieredHNSW_BatchIterator::getNextResults() is called and hnsw_iterator is not initialized, it + // retrieves the blob from flat_iterator : VecSimBatchIterator(nullptr, queryParams ? queryParams->timeoutCtx : nullptr, std::move(allocator)), index(index), flat_results(this->allocator), hnsw_results(this->allocator), @@ -1137,8 +1120,8 @@ VecSimQueryReply *TieredHNSWIndex::TieredHNSW_BatchIterator: auto hnsw_code = VecSim_QueryReply_OK; if (this->hnsw_iterator == UNINITIALIZED) { - // First call to getNextResults. The call to the BF iterator will include calculating - // all the distances and access the BF index. We take the lock on this call. + // First call to getNextResults. The call to the BF iterator will include calculating all + // the distances and access the BF index. We take the lock on this call. this->index->flatIndexGuard.lock_shared(); auto cur_flat_results = this->flat_iterator->getNextResults(n_res, BY_SCORE_THEN_ID); this->index->flatIndexGuard.unlock_shared(); @@ -1148,8 +1131,8 @@ VecSimQueryReply *TieredHNSWIndex::TieredHNSW_BatchIterator: } this->flat_results.swap(cur_flat_results->results); VecSimQueryReply_Free(cur_flat_results); - // We also take the lock on the main index on the first call to getNextResults, and we - // hold it until the iterator is depleted or freed. + // We also take the lock on the main index on the first call to getNextResults, and we hold + // it until the iterator is depleted or freed. this->index->mainIndexGuard.lock_shared(); this->hnsw_iterator = this->index->backendIndex->newBatchIterator( this->flat_iterator->getQueryBlob(), queryParams); @@ -1171,10 +1154,10 @@ VecSimQueryReply *TieredHNSWIndex::TieredHNSW_BatchIterator: VecSimQueryReply_Free(tail); if (!isMulti) { - // On single-value indexes, duplicates will never appear in the hnsw results - // before they appear in the flat results (at the same time or later if the - // approximation misses) so we don't need to try and filter the flat results - // (and recheck conditions). + // On single-value indexes, duplicates will never appear in the hnsw results before + // they appear in the flat results (at the same time or later if the approximation + // misses) so we don't need to try and filter the flat results (and recheck + // conditions). break; } else { // On multi-value indexes, the flat results may contain results that are already @@ -1188,9 +1171,9 @@ VecSimQueryReply *TieredHNSWIndex::TieredHNSW_BatchIterator: auto tail = this->hnsw_iterator->getNextResults(n_res - this->hnsw_results.size(), BY_SCORE_THEN_ID); hnsw_code = tail->code; // Set the hnsw_results code to the last `getNextResults` code. - // New batch may contain better results than the previous batch, so we need to - // merge. We don't expect duplications (hence the ), as the iterator - // guarantees that no result is returned twice. + // New batch may contain better results than the previous batch, so we need to merge. + // We don't expect duplications (hence the ), as the iterator guarantees that + // no result is returned twice. VecSimQueryResultContainer cur_hnsw_results(this->allocator); merge_results(cur_hnsw_results, this->hnsw_results, tail->results, n_res); VecSimQueryReply_Free(tail); @@ -1267,15 +1250,13 @@ TieredHNSWIndex::TieredHNSW_BatchIterator::compute_current_b auto [from_hnsw, from_flat] = p; if (!isMultiValue) { - // If we're on a single-value index, update the set of results returned from the FLAT - // index before popping them, to prevent them to be returned from the HNSW index in - // later batches. + // If we're on a single-value index, update the set of results returned from the FLAT index + // before popping them, to prevent them to be returned from the HNSW index in later batches. for (size_t i = 0; i < from_flat; ++i) { this->returned_results_set.insert(this->flat_results[i].id); } } else { - // If we're on a multi-value index, update the set of results returned (from - // `batch_res`) + // If we're on a multi-value index, update the set of results returned (from `batch_res`) for (size_t i = 0; i < batch_res->results.size(); ++i) { this->returned_results_set.insert(batch_res->results[i].id); } From ab6bfde79bdae70d61eca305eefbf48dd70c380b Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Mon, 14 Sep 2026 08:49:19 -0700 Subject: [PATCH 45/64] add missing file --- src/VecSim/utils/scoped_locks.h | 58 +++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 src/VecSim/utils/scoped_locks.h diff --git a/src/VecSim/utils/scoped_locks.h b/src/VecSim/utils/scoped_locks.h new file mode 100644 index 000000000..234047a9b --- /dev/null +++ b/src/VecSim/utils/scoped_locks.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ + +#pragma once + +#include +#include + +// A lockable resource, so that different mutex-like primitives (a shared_mutex, HNSW's own +// data guard, or nothing at all) can be composed under one RAII type below. +class Lockable { +public: + virtual void lock() const = 0; + virtual void unlock() const = 0; + virtual ~Lockable() = default; +}; + +// Adapts a std::shared_mutex, locked/unlocked for shared (read) access, to Lockable. +class SharedMutexLockable : public Lockable { + std::shared_mutex &guard; + +public: + explicit SharedMutexLockable(std::shared_mutex &guard) : guard(guard) {} + void lock() const override { guard.lock_shared(); } + void unlock() const override { guard.unlock_shared(); } +}; + +// RAII acquisition of zero, one or two Lockables, so callers can plug in whatever locking +// (or none) they need without repeating the lock/unlock bookkeeping. +class ScopedLocks { + std::array locks{}; + size_t count = 0; + +public: + ScopedLocks() = default; + explicit ScopedLocks(const Lockable &a) : locks{&a, nullptr}, count(1) { a.lock(); } + ScopedLocks(const Lockable &a, const Lockable &b) : locks{&a, &b}, count(2) { + a.lock(); + b.lock(); + } + ScopedLocks(ScopedLocks &&other) noexcept : locks(other.locks), count(other.count) { + other.count = 0; + } + ScopedLocks(const ScopedLocks &) = delete; + ScopedLocks &operator=(ScopedLocks &&) = delete; + ~ScopedLocks() { + // Unlock in reverse acquisition order. + for (size_t i = count; i-- > 0;) { + locks[i]->unlock(); + } + } +}; From cc9b6661ad811b335ba3189f9f442189c974e29e Mon Sep 17 00:00:00 2001 From: ethanglaser Date: Mon, 14 Sep 2026 21:16:28 -0700 Subject: [PATCH 46/64] use SVS rc2 binaries and latest submodule --- cmake/svs.cmake | 6 +++--- deps/ScalableVectorSearch | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index f6aebd87a..c1c04863b 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -64,7 +64,7 @@ if(USE_SVS) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.31" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "21.0") if(GLIBCXX_VERSION VERSION_GREATER_EQUAL "11") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc1-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc2-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") else() message(STATUS "libstdc++ >= GCC 11 is required for Clang SVS binaries - disabling SVS_SHARED_LIB") set(SVS_SHARED_LIB OFF) @@ -76,9 +76,9 @@ if(USE_SVS) else() if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.28") if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "14.0") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc1-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc2-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") else() - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc1-reduced.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc2-reduced.tar.gz" CACHE STRING "SVS URL") endif() else() message(STATUS "GLIBC >= 2.26 is required for GCC SVS binaries - disabling SVS_SHARED_LIB") diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch index 389597b59..3ee890d6d 160000 --- a/deps/ScalableVectorSearch +++ b/deps/ScalableVectorSearch @@ -1 +1 @@ -Subproject commit 389597b59cbfe87c6887da262673ef386261947e +Subproject commit 3ee890d6d52102e979ebd4331823b6236ec17a56 From 86a26f15dd4b6228debbf8bb7022924cca7a04c2 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 14 Sep 2026 23:22:52 -0700 Subject: [PATCH 47/64] fix iterator --- src/VecSim/algorithms/svs/svs_tiered.h | 60 +++++++------------------- 1 file changed, 15 insertions(+), 45 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index ea262cbb8..7e91a98d1 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -305,19 +305,14 @@ class TieredSVSIndex : public VecSimTieredIndex { VecSimBatchIterator *flat_iterator; VecSimBatchIterator *svs_iterator; - // On single value indices, this set holds the IDs of the results that were returned from - // the flat buffer. - // On multi value indices, this set holds the IDs of all the results that were returned. - // The difference between the two cases is that on multi value indices, the same ID can - // appear in both indexes and results with different scores, and therefore we can't tell in - // advance when we expect a possibility of a duplicate. - // On single value indices, a duplicate may appear at the same batch (and we will handle it - // when merging the results) Or it may appear in a different batches, first from the flat - // buffer and then from the SVS, in the cases where a better result if found later in SVS - // because of the approximate nature of the algorithm. + // IDs of all the results returned so far, so that neither tier can yield one twice. + // Both directions have to be covered: nothing freezes the backend for the iterator's + // lifetime, so an insert job can publish a label into SVS and only then drop it from the + // flat buffer, while the flat iterator still holds it in the snapshot it computed on its + // first call. Either tier can therefore surface a label first. vecsim_stl::unordered_set returned_results_set; - VecSimQueryReply *compute_current_batch(size_t n_res, bool isMultiValue) { + VecSimQueryReply *compute_current_batch(size_t n_res) { // Merge results // This call will update `svs_res` and `bf_res` to point to the end of the merged // results. @@ -327,33 +322,18 @@ class TieredSVSIndex : public VecSimTieredIndex { auto [from_svs, from_flat] = merge_results(batch_res->results, svs_results, flat_results, n_res); - if (!isMultiValue) { - // If we're on a single-value index, update the set of results returned from the - // FLAT index before popping them, to prevent them to be returned from the SVS index - // in later batches. - for (size_t i = 0; i < from_flat; ++i) { - this->returned_results_set.insert(this->flat_results[i].id); - } - } else { - // If we're on a multi-value index, update the set of results returned (from - // `batch_res`) - for (size_t i = 0; i < batch_res->results.size(); ++i) { - this->returned_results_set.insert(batch_res->results[i].id); - } + for (size_t i = 0; i < batch_res->results.size(); ++i) { + this->returned_results_set.insert(batch_res->results[i].id); } // Update results flat_results.erase(flat_results.begin(), flat_results.begin() + from_flat); svs_results.erase(svs_results.begin(), svs_results.begin() + from_svs); - // clean up the results - // On multi-value indexes, one (or both) results lists may contain results that are - // already returned form the other list (with a different score). We need to filter them - // out. - if (isMultiValue) { - filter_irrelevant_results(this->flat_results); - filter_irrelevant_results(this->svs_results); - } + // Either list may still hold a label just returned from the other one (with a different + // score), so drop those before the next batch. + filter_irrelevant_results(this->flat_results); + filter_irrelevant_results(this->svs_results); // Return current batch return batch_res; @@ -422,7 +402,6 @@ class TieredSVSIndex : public VecSimTieredIndex { VecSimQueryReply *getNextResults(size_t n_res, VecSimQueryReply_Order order) override { auto svs_code = VecSim_QueryReply_OK; - const bool isMulti = this->index->backendIndex->isMultiValue(); if (svs_iterator == nullptr) { // first call // First call to getNextResults. The call to the BF iterator will include // calculating all the distances and access the BF index. We take the lock on this @@ -453,17 +432,8 @@ class TieredSVSIndex : public VecSimTieredIndex { tail->results.end()); VecSimQueryReply_Free(tail); - if (!isMulti) { - // On single-value indexes, duplicates will never appear in the hnsw results - // before they appear in the flat results (at the same time or later if the - // approximation misses) so we don't need to try and filter the flat results - // (and recheck conditions). - break; - } else { - // On multi-value indexes, the flat results may contain results that are - // already returned from the hnsw index. We need to filter them out. - filter_irrelevant_results(this->flat_results); - } + // The flat results may contain labels already returned from the SVS index. + filter_irrelevant_results(this->flat_results); } while (svs_results.size() < n_res && svs_iterator != depleted() && @@ -489,7 +459,7 @@ class TieredSVSIndex : public VecSimTieredIndex { } VecSimQueryReply *batch; - batch = compute_current_batch(n_res, isMulti); + batch = compute_current_batch(n_res); if (order == BY_ID) { sort_results_by_id(batch); From 964a913fec5c99875bba4741b4b709a66771881d Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Tue, 15 Sep 2026 00:27:37 -0700 Subject: [PATCH 48/64] relabelVector: switch check to concurent; rewrite impl call logic --- cmake/svs.cmake | 2 +- src/VecSim/algorithms/svs/svs.h | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index d777bc208..b9aef3431 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -126,7 +126,7 @@ if(USE_SVS) # so whether it is available depends on which SVS this build ended up with rather than on the # platform. Detect it in the header instead of assuming: it is a method, not a file, so this # greps the header the LVQ check would have tested for existence. - set(SVS_DYNAMIC_INDEX_HEADER "svs/index/vamana/dynamic_index.h") + set(SVS_DYNAMIC_INDEX_HEADER "svs/concurrent/dynamic_index.h") set(SVS_HAS_REPLACE_EXTERNAL_ID 0) if(EXISTS "${svs_SOURCE_DIR}/include/${SVS_DYNAMIC_INDEX_HEADER}") file(READ "${svs_SOURCE_DIR}/include/${SVS_DYNAMIC_INDEX_HEADER}" SVS_DYNAMIC_INDEX_SRC) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 2dc449e06..e41f5525d 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -610,16 +610,19 @@ class SVSIndex : public VecSimIndexAbstract, fl if (old_label == new_label) { return VecSimRelabel_SameLabel; } - // `isLabelExists` also covers the index that never held a vector, where `impl_` has not - // been created yet and so trivially holds nothing. - if (!isLabelExists(old_label)) { + + std::shared_lock mutation_lock(this->implMutationGuard_); + auto impl = getImpl(); + + // A null impl_ is an index that never held a vector, so it holds neither label. + if (!impl || !impl->has_id(old_label)) { return VecSimRelabel_OldLabelMissing; } - if (isLabelExists(new_label)) { + if (impl->has_id(new_label)) { return VecSimRelabel_NewLabelTaken; } - impl_->replace_external_id(old_label, new_label); + impl->replace_external_id(old_label, new_label); return VecSimRelabel_OK; } #endif // HAVE_SVS_REPLACE_EXTERNAL_ID From aedb77c87e756e3e46e810d7d60c2f5ea53033a6 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 15 Sep 2026 03:30:47 -0700 Subject: [PATCH 49/64] harmonize svs and hnsw tiered job types --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 17 ++++------------- src/VecSim/algorithms/svs/svs_tiered.h | 15 +++++---------- src/VecSim/vec_sim_tiered_index.h | 22 ++++++++++++++++++++++ 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index 6d2aeac1c..4ff54f44e 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -16,23 +16,14 @@ #include "hnsw.h" #include "VecSim/index_factories/hnsw_factory.h" -/** - * Definition of a job that inserts a new vector from flat into HNSW Index. - */ -struct HNSWInsertJob : public AsyncJob { - labelType label; - idType id; - - HNSWInsertJob(std::shared_ptr allocator, labelType label_, idType id_, - JobCallback insertCb, VecSimIndex *index_) - : AsyncJob(allocator, HNSW_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} -}; +using HNSWInsertJob = TieredInsertJob; /** * Definition of a job that swaps last id with a deleted id in HNSW Index after delete operation. */ struct HNSWSwapJob : public VecsimBaseObject { idType deleted_id; + std::atomic_int pending_repair_jobs_counter; // number of repair jobs left to complete before this job // is ready to be executed (atomic counter). @@ -53,14 +44,14 @@ static const size_t MAX_PENDING_SWAP_JOBS_THRESHOLD = 100000; * Definition of a job that repairs a certain node's connection in HNSW Index after delete * operation. */ -struct HNSWRepairJob : public AsyncJob { +struct HNSWRepairJob : public TieredJob { idType node_id; unsigned short level; vecsim_stl::vector associatedSwapJobs; HNSWRepairJob(std::shared_ptr allocator, idType id_, unsigned short level_, JobCallback repairCb, VecSimIndex *index_, HNSWSwapJob *swapJob) - : AsyncJob(allocator, HNSW_REPAIR_NODE_CONNECTIONS_JOB, repairCb, index_), node_id(id_), + : TieredJob(allocator, repairCb, index_), node_id(id_), level(level_), // Insert the first swap job from which this repair job was created. associatedSwapJobs(1, swapJob, this->allocator) {} diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 83dea01e4..d97867662 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -14,29 +14,24 @@ #include #include -/** - * Definition of a job that inserts a new vector from flat into SVS Index. - */ -struct SVSInsertJob : public AsyncJob { - labelType label; - idType id; +struct SVSInsertJob : public TieredInsertJob { std::atomic executing{false}; SVSInsertJob(std::shared_ptr allocator, labelType label_, idType id_, JobCallback insertCb, VecSimIndex *index_) - : AsyncJob(allocator, SVS_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} + : TieredInsertJob(allocator, label_, id_, insertCb, index_) {} }; /** * Definition of a job that launches partial consolidation on SVS Index. */ -struct SVSConsolidateJob : public AsyncJob { +struct SVSConsolidateJob : public TieredJob { std::vector labels; SVSConsolidateJob(std::shared_ptr allocator, const std::vector &labels_, JobCallback insertCb, VecSimIndex *index_) - : AsyncJob(allocator, SVS_CONSOLIDATE_JOB, insertCb, index_), labels(labels_) {} + : TieredJob(allocator, insertCb, index_), labels(labels_) {} }; /** @@ -579,7 +574,7 @@ class TieredSVSIndex : public VecSimTieredIndex { static void executeInsertJobWrapper(AsyncJob *job) { auto *insert_job = static_cast(job); auto *job_index = static_cast *>(insert_job->index); - InsertJobOutcome outcome; + InsertJobOutcome outcome = InsertJobOutcome::Completed; { // prevent parallel execution with index initilizing job std::shared_lock lock(job_index->updateJobMutex); diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 9a2ebb2ce..19751dbbb 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -41,6 +41,28 @@ struct AsyncJob : public VecsimBaseObject { isValid(true) {} }; +/** + * Common base for asynchronous jobs submitted by tiered indexes. + */ +template +struct TieredJob : public AsyncJob { + TieredJob(std::shared_ptr allocator, JobCallback callback, VecSimIndex *index) + : AsyncJob(allocator, Type, callback, index) {} +}; + +/** + * A job that inserts a new vector from the flat buffer to the backend. + */ +template +struct TieredInsertJob : public TieredJob { + labelType label; + idType id; + + TieredInsertJob(std::shared_ptr allocator, labelType label_, idType id_, + JobCallback insertCb, VecSimIndex *index_) + : TieredJob(allocator, insertCb, index_), label(label_), id(id_) {} +}; + // All read operations (including KNN, range, batch iterators and get-distance-from) are guaranteed // to consider all vectors that were added to the index before the query was submitted. The results // may include vectors that were added after the query was submitted, with no guarantees. From 0b68c823d106381f23757a2df57952c064a7afe3 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Tue, 15 Sep 2026 05:25:42 -0700 Subject: [PATCH 50/64] Revert "harmonize svs and hnsw tiered job types" This reverts commit aedb77c87e756e3e46e810d7d60c2f5ea53033a6. --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 17 +++++++++++++---- src/VecSim/algorithms/svs/svs_tiered.h | 15 ++++++++++----- src/VecSim/vec_sim_tiered_index.h | 22 ---------------------- 3 files changed, 23 insertions(+), 31 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index 4ff54f44e..6d2aeac1c 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -16,14 +16,23 @@ #include "hnsw.h" #include "VecSim/index_factories/hnsw_factory.h" -using HNSWInsertJob = TieredInsertJob; +/** + * Definition of a job that inserts a new vector from flat into HNSW Index. + */ +struct HNSWInsertJob : public AsyncJob { + labelType label; + idType id; + + HNSWInsertJob(std::shared_ptr allocator, labelType label_, idType id_, + JobCallback insertCb, VecSimIndex *index_) + : AsyncJob(allocator, HNSW_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} +}; /** * Definition of a job that swaps last id with a deleted id in HNSW Index after delete operation. */ struct HNSWSwapJob : public VecsimBaseObject { idType deleted_id; - std::atomic_int pending_repair_jobs_counter; // number of repair jobs left to complete before this job // is ready to be executed (atomic counter). @@ -44,14 +53,14 @@ static const size_t MAX_PENDING_SWAP_JOBS_THRESHOLD = 100000; * Definition of a job that repairs a certain node's connection in HNSW Index after delete * operation. */ -struct HNSWRepairJob : public TieredJob { +struct HNSWRepairJob : public AsyncJob { idType node_id; unsigned short level; vecsim_stl::vector associatedSwapJobs; HNSWRepairJob(std::shared_ptr allocator, idType id_, unsigned short level_, JobCallback repairCb, VecSimIndex *index_, HNSWSwapJob *swapJob) - : TieredJob(allocator, repairCb, index_), node_id(id_), + : AsyncJob(allocator, HNSW_REPAIR_NODE_CONNECTIONS_JOB, repairCb, index_), node_id(id_), level(level_), // Insert the first swap job from which this repair job was created. associatedSwapJobs(1, swapJob, this->allocator) {} diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index d97867662..83dea01e4 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -14,24 +14,29 @@ #include #include -struct SVSInsertJob : public TieredInsertJob { +/** + * Definition of a job that inserts a new vector from flat into SVS Index. + */ +struct SVSInsertJob : public AsyncJob { + labelType label; + idType id; std::atomic executing{false}; SVSInsertJob(std::shared_ptr allocator, labelType label_, idType id_, JobCallback insertCb, VecSimIndex *index_) - : TieredInsertJob(allocator, label_, id_, insertCb, index_) {} + : AsyncJob(allocator, SVS_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} }; /** * Definition of a job that launches partial consolidation on SVS Index. */ -struct SVSConsolidateJob : public TieredJob { +struct SVSConsolidateJob : public AsyncJob { std::vector labels; SVSConsolidateJob(std::shared_ptr allocator, const std::vector &labels_, JobCallback insertCb, VecSimIndex *index_) - : TieredJob(allocator, insertCb, index_), labels(labels_) {} + : AsyncJob(allocator, SVS_CONSOLIDATE_JOB, insertCb, index_), labels(labels_) {} }; /** @@ -574,7 +579,7 @@ class TieredSVSIndex : public VecSimTieredIndex { static void executeInsertJobWrapper(AsyncJob *job) { auto *insert_job = static_cast(job); auto *job_index = static_cast *>(insert_job->index); - InsertJobOutcome outcome = InsertJobOutcome::Completed; + InsertJobOutcome outcome; { // prevent parallel execution with index initilizing job std::shared_lock lock(job_index->updateJobMutex); diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 19751dbbb..9a2ebb2ce 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -41,28 +41,6 @@ struct AsyncJob : public VecsimBaseObject { isValid(true) {} }; -/** - * Common base for asynchronous jobs submitted by tiered indexes. - */ -template -struct TieredJob : public AsyncJob { - TieredJob(std::shared_ptr allocator, JobCallback callback, VecSimIndex *index) - : AsyncJob(allocator, Type, callback, index) {} -}; - -/** - * A job that inserts a new vector from the flat buffer to the backend. - */ -template -struct TieredInsertJob : public TieredJob { - labelType label; - idType id; - - TieredInsertJob(std::shared_ptr allocator, labelType label_, idType id_, - JobCallback insertCb, VecSimIndex *index_) - : TieredJob(allocator, insertCb, index_), label(label_), id(id_) {} -}; - // All read operations (including KNN, range, batch iterators and get-distance-from) are guaranteed // to consider all vectors that were added to the index before the query was submitted. The results // may include vectors that were added after the query was submitted, with no guarantees. From 90b22f8ae8fda861a62b10fbd35af96521166bce Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Tue, 15 Sep 2026 07:32:43 -0700 Subject: [PATCH 51/64] adopt relabelVector for concuren svs --- src/VecSim/algorithms/svs/svs.h | 15 +- src/VecSim/algorithms/svs/svs_tiered.h | 192 ++++++++++++++++++------- 2 files changed, 154 insertions(+), 53 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index e41f5525d..3a6989ae5 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -615,15 +615,20 @@ class SVSIndex : public VecSimIndexAbstract, fl auto impl = getImpl(); // A null impl_ is an index that never held a vector, so it holds neither label. - if (!impl || !impl->has_id(old_label)) { + if (!impl) { return VecSimRelabel_OldLabelMissing; } - if (impl->has_id(new_label)) { + + // SVS checks both ends under its own translator lock + switch (impl->replace_external_id(old_label, new_label)) { + case svs::concurrent::ReplaceExternalIdResult::Ok: + return VecSimRelabel_OK; + case svs::concurrent::ReplaceExternalIdResult::NewIdExists: return VecSimRelabel_NewLabelTaken; + case svs::concurrent::ReplaceExternalIdResult::OldIdMissing: + break; } - - impl->replace_external_id(old_label, new_label); - return VecSimRelabel_OK; + return VecSimRelabel_OldLabelMissing; } #endif // HAVE_SVS_REPLACE_EXTERNAL_ID diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 83dea01e4..4d0ebe2cb 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -32,6 +32,7 @@ struct SVSInsertJob : public AsyncJob { */ struct SVSConsolidateJob : public AsyncJob { std::vector labels; + std::atomic executing{false}; SVSConsolidateJob(std::shared_ptr allocator, const std::vector &labels_, JobCallback insertCb, @@ -276,8 +277,9 @@ class TieredSVSIndex : public VecSimTieredIndex { idType currInvalidJobId; // A unique arbitrary identifier for accessing invalid jobs std::mutex invalidJobsLookupGuard; - vecsim_stl::unordered_set pendingConsolidateJobs; - std::mutex consolidateJobsGuard; + vecsim_stl::unordered_map> + labelToConsolidateJobs; + mutable std::shared_mutex consolidateJobsGuard; size_t flat_buffer_bound; @@ -641,11 +643,21 @@ class TieredSVSIndex : public VecSimTieredIndex { std::shared_lock lock(index->updateJobMutex); auto svs_index = index->GetSVSIndex(); svs_index->setParallelism(1); - svs_index->consolidate(consolidate_job->labels); + + bool valid = false; { - std::lock_guard jobs_lock(index->consolidateJobsGuard); - index->pendingConsolidateJobs.erase(job); + std::shared_lock flat_lock(index->flatIndexGuard); + valid = consolidate_job->isValid; + if (valid) { + consolidate_job->executing.store(true, std::memory_order_release); + } + } + if (valid) { + svs_index->consolidate(consolidate_job->labels); + // Cleared before the registry lock is needed again, so a waiter cannot deadlock us. + consolidate_job->executing.store(false, std::memory_order_release); } + index->forgetConsolidateJob(consolidate_job); delete job; } @@ -692,23 +704,79 @@ class TieredSVSIndex : public VecSimTieredIndex { } void scheduleSVSIndexConsolidate(labelType label) { - AsyncJob *new_consolidate_job = new (this->allocator) + auto *new_consolidate_job = new (this->allocator) SVSConsolidateJob(this->allocator, {label}, SVSIndexConsolidateWrapper, this); { - std::lock_guard lock(this->consolidateJobsGuard); - this->pendingConsolidateJobs.insert(new_consolidate_job); + std::lock_guard lock(this->consolidateJobsGuard); + auto it = this->labelToConsolidateJobs.find(label); + if (it != this->labelToConsolidateJobs.end()) { + it->second.push_back(new_consolidate_job); + } else { + vecsim_stl::vector jobs(1, new_consolidate_job, + this->allocator); + this->labelToConsolidateJobs.insert({label, std::move(jobs)}); + } } // Insert job to the queue. this->submitSingleJob(new_consolidate_job); } private: - idType setAndSaveInvalidJob(SVSInsertJob *job) { - // wait until insertion finishes + void forgetConsolidateJob(SVSConsolidateJob *job) { + std::lock_guard lock(this->consolidateJobsGuard); + for (auto label : job->labels) { + auto it = this->labelToConsolidateJobs.find(label); + if (it == this->labelToConsolidateJobs.end()) { + continue; + } + auto &jobs = it->second; + jobs.erase(std::remove(jobs.begin(), jobs.end(), job), jobs.end()); + if (jobs.empty()) { + this->labelToConsolidateJobs.erase(it); + } + } + } + // Caller must hold flatIndexGuard exclusive + std::vector takeOverConsolidateOf(labelType label) { + std::vector taken_over; + vecsim_stl::vector running(this->allocator); + { + std::shared_lock lock(this->consolidateJobsGuard); + auto it = this->labelToConsolidateJobs.find(label); + if (it == this->labelToConsolidateJobs.end()) { + return taken_over; + } + for (auto *job : it->second) { + if (!job->isValid) { + continue; // already taken over by an earlier call + } + if (job->executing.load(std::memory_order_acquire)) { + running.push_back(job); + } else { + job->isValid = false; + taken_over.insert(taken_over.end(), job->labels.begin(), job->labels.end()); + } + } + } + for (auto *job : running) { + while (job->executing.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + } + return taken_over; + } + + // Wait until the job leaves its publish window + // Safe to call while holding flatIndexGuard exclusive + static void waitForInsertJob(SVSInsertJob *job) { while (job->executing.load(std::memory_order_acquire)) { std::this_thread::yield(); } + } + + idType setAndSaveInvalidJob(SVSInsertJob *job) { + waitForInsertJob(job); this->invalidJobsLookupGuard.lock(); job->isValid = false; idType curInvalidId = currInvalidJobId++; @@ -870,7 +938,7 @@ class TieredSVSIndex : public VecSimTieredIndex { : Base(svs_index, bf_index, tiered_index_params, allocator), uncompletedJobs(this->allocator), labelToInsertJobs(this->allocator), invalidJobs(this->allocator), currInvalidJobId(0), - pendingConsolidateJobs(this->allocator) { + labelToConsolidateJobs(this->allocator) { const auto &tiered_svs_params = tiered_index_params.specificParams.tieredSVSParams; // If flatBufferLimit is not initialized (0), use the default update threshold. @@ -912,8 +980,10 @@ class TieredSVSIndex : public VecSimTieredIndex { } // Delete all the pending consolidate jobs - for (auto *job : this->pendingConsolidateJobs) { - delete job; + for (auto &jobs : this->labelToConsolidateJobs) { + for (auto *job : jobs.second) { + delete job; + } } } @@ -1179,12 +1249,6 @@ class TieredSVSIndex : public VecSimTieredIndex { * Reports `Unsupported` when the backend holds the label and cannot move it, which is the * case when built against an SVS without `replace_external_id`. All-or-nothing: on any code * other than `VecSimRelabel_OK` both tiers are untouched. - * - * `updateJobMutex` is taken first, in the order `updateSVSIndex` takes its own locks. Holding - * it is what makes this correct rather than merely serialised: an update job snapshots the - * buffer's labels *by value* and afterwards reconciles only id swaps and deletions, so a - * rename landing inside its window would be invisible to it and the vector would reach the - * backend under the old label. */ VecSimRelabelCode relabelVector(labelType old_label, labelType new_label) override { if (old_label == new_label) { @@ -1192,39 +1256,71 @@ class TieredSVSIndex : public VecSimTieredIndex { } auto *svs_index = GetSVSIndex(); - // Taken together through `std::lock`'s back-off rather than one after another, because - // the orders in this class disagree: nearly everything acquires flat before main, but the - // in-place add path takes `mainIndexGuard` before `flatIndexGuard` while the backend is - // still empty. Acquiring in sequence would hold one guard while blocking on another and - // close a cycle with that path; acquiring them together cannot, whichever order the other - // side uses. Same idiom that path already uses for its own two locks. - std::scoped_lock lock(this->updateJobMutex, this->flatIndexGuard, this->mainIndexGuard); - - const bool in_flat = this->frontendIndex->isLabelExists(old_label); - const bool in_backend = svs_index->isLabelExists(old_label); - if (!in_flat && !in_backend) { - return VecSimRelabel_OldLabelMissing; + std::shared_lock lock(this->updateJobMutex); + + bool flat_holds_old = false; + std::vector taken_over; + { + std::lock_guard flat_lock{this->flatIndexGuard}; + + flat_holds_old = this->frontendIndex->isLabelExists(old_label); + if (!flat_holds_old && !svs_index->isLabelExists(old_label)) { + return VecSimRelabel_OldLabelMissing; + } + if (this->frontendIndex->isLabelExists(new_label) || + svs_index->isLabelExists(new_label)) { + return VecSimRelabel_NewLabelTaken; + } + + auto pending = this->labelToInsertJobs.find(old_label); + if (pending != this->labelToInsertJobs.end()) { + auto jobs = std::move(pending->second); + this->labelToInsertJobs.erase(pending); + for (auto *job : jobs) { + waitForInsertJob(job); + job->label = new_label; + } + this->labelToInsertJobs.emplace(new_label, std::move(jobs)); + } + + // isLabelExists() said new_label is free, but SVS keeps the translator entry of a + // soft-deleted label until its consolidate job runs, and would refuse the remap. + taken_over = takeOverConsolidateOf(new_label); + + if (flat_holds_old) { + const VecSimRelabelCode flat_ret = + this->frontendIndex->relabelVector(old_label, new_label); + assert(flat_ret == VecSimRelabel_OK && + "the buffer just reported holding this label"); + UNUSED(flat_ret); + } } - if (this->frontendIndex->isLabelExists(new_label) || svs_index->isLabelExists(new_label)) { - return VecSimRelabel_NewLabelTaken; + + // Do the taken-over job's, + // the remap below isn't refused by a leftover translator entry + if (!taken_over.empty()) { + svs_index->consolidate(taken_over); } - // The backend goes first because it is the tier that can refuse; refusing after the - // buffer had already moved would leave the label half applied. - if (in_backend) { - const VecSimRelabelCode backend_ret = - this->backendIndex->relabelVector(old_label, new_label); - if (backend_ret != VecSimRelabel_OK) { - return backend_ret; + const VecSimRelabelCode backend_ret = + this->backendIndex->relabelVector(old_label, new_label); + if (backend_ret == VecSimRelabel_NewLabelTaken) { + // Only reachable if new_label was added concurrently with this rename + std::lock_guard flat_lock{this->flatIndexGuard}; + auto pending = this->labelToInsertJobs.find(new_label); + if (pending != this->labelToInsertJobs.end()) { + auto jobs = std::move(pending->second); + this->labelToInsertJobs.erase(pending); + for (auto *job : jobs) { + waitForInsertJob(job); + job->label = old_label; + } + this->labelToInsertJobs.emplace(old_label, std::move(jobs)); } - } - if (in_flat) { - const VecSimRelabelCode flat_ret = - this->frontendIndex->relabelVector(old_label, new_label); -#ifdef BUILD_TESTS - assert(flat_ret == VecSimRelabel_OK && "the buffer just reported holding this label"); -#endif - UNUSED(flat_ret); + if (flat_holds_old) { + this->frontendIndex->relabelVector(new_label, old_label); + } + return VecSimRelabel_NewLabelTaken; } return VecSimRelabel_OK; } From ffab81869d7ab3a97787b1fb7cab711dca94a451 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 01:24:14 -0700 Subject: [PATCH 52/64] move hnsw/svs tierd common logic to the base class --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 120 +++-------------------- src/VecSim/algorithms/svs/svs_tiered.h | 110 +++------------------ src/VecSim/vec_sim_tiered_index.h | 103 ++++++++++++++++++- tests/unit/test_hnsw_tiered.cpp | 10 +- 4 files changed, 137 insertions(+), 206 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index 3132e9bc7..5a7760875 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -24,13 +24,10 @@ /** * Definition of a job that inserts a new vector from flat into HNSW Index. */ -struct HNSWInsertJob : public AsyncJob { - labelType label; - idType id; - +struct HNSWInsertJob : public TieredInsertJob { HNSWInsertJob(std::shared_ptr allocator, labelType label_, idType id_, JobCallback insertCb, VecSimIndex *index_) - : AsyncJob(allocator, HNSW_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} + : TieredInsertJob(allocator, HNSW_INSERT_VECTOR_JOB, label_, id_, insertCb, index_) {} }; /** @@ -81,17 +78,10 @@ template class TieredHNSWIndex : public VecSimTieredIndex { private: /// Mappings from id/label to associated jobs, for invalidating and update ids if necessary. - // In MULTI, we can have more than one insert job pending per label. // **This map is protected with the flat buffer lock** - vecsim_stl::unordered_map> labelToInsertJobs; vecsim_stl::unordered_map> idToRepairJobs; vecsim_stl::unordered_map idToSwapJob; - // A mapping to hold invalid jobs, so we can dispose them upon index deletion. - vecsim_stl::unordered_map invalidJobs; - idType currInvalidJobId; // A unique arbitrary identifier for accessing invalid jobs - std::mutex invalidJobsLookupGuard; - // This threshold is tested upon deleting a label from HNSW, and once the number of deleted // vectors reached this limit, we apply swap jobs *only for vectors that has no more pending // repair jobs*, and are ready to be removed from the graph. @@ -172,13 +162,6 @@ class TieredHNSWIndex : public VecSimTieredIndex { inline HNSWIndex *getHNSWIndex() const; - // Helper function for deleting a vector from the flat buffer (after it has already been - // ingested into HNSW or deleted). This includes removing the corresponding insert job from the - // label-to-insert-jobs lookup. Also, since deletion a vector triggers swapping of the - // internal last id with the deleted vector id, here we update the pending insert job(s) for the - // last id (if needed). This should be called while *flat lock is held* (exclusive lock). - void updateInsertJobInternalId(idType prev_id, idType new_id, labelType label); - // Helper function for performing in place mark delete of vector(s) associated with a label // and creating the appropriate repair jobs for the effected connections. This should be called // while *HNSW shared lock is held* (shared locked). @@ -197,11 +180,6 @@ class TieredHNSWIndex : public VecSimTieredIndex { void insertVectorToHNSW(HNSWIndex *hnsw_index, labelType label, const void *blob); - // Set an insert/repair job as invalid, put the job pointer in the invalid jobs lookup under - // the current available id, increase it and return it (while holding invalidJobsLookupGuard). - // Returns the id that the job was stored under (to be set in the job id field). - idType setAndSaveInvalidJob(AsyncJob *job); - // Handle deletion of vector inplace considering that async deletion might occurred beforehand. int deleteLabelFromHNSWInplace(labelType label); @@ -540,22 +518,6 @@ int TieredHNSWIndex::deleteLabelFromHNSW(labelType label) { return internal_ids.size(); } -template -void TieredHNSWIndex::updateInsertJobInternalId(idType prev_id, idType new_id, - labelType label) { - // Update the pending job id, due to a swap that was caused after the removal of new_id. - assert(new_id != INVALID_ID && prev_id != INVALID_ID); - auto it = this->labelToInsertJobs.find(label); - if (it != this->labelToInsertJobs.end()) { - // There is a pending job for the label of the swapped last id - update its id. - for (HNSWInsertJob *job_it : it->second) { - if (job_it->id == prev_id) { - job_it->id = new_id; - } - } - } -} - template template void TieredHNSWIndex::insertVectorToHNSW( @@ -625,16 +587,6 @@ void TieredHNSWIndex::insertVectorToHNSW( } } -template -idType TieredHNSWIndex::setAndSaveInvalidJob(AsyncJob *job) { - this->invalidJobsLookupGuard.lock(); - job->isValid = false; - idType curInvalidId = currInvalidJobId++; - this->invalidJobs.insert({curInvalidId, job}); - this->invalidJobsLookupGuard.unlock(); - return curInvalidId; -} - template int TieredHNSWIndex::deleteLabelFromHNSWInplace(labelType label) { auto *hnsw_index = this->getHNSWIndex(); @@ -692,40 +644,7 @@ void TieredHNSWIndex::executeInsertJob(HNSWInsertJob *job) { #endif // Remove the vector and the insert job from the flat buffer. - this->flatIndexGuard.lock(); - // The job might have been invalidated due to overwrite in the meantime. In this case, - // it was already deleted and the job has been evicted. Otherwise, we need to do it now. - if (job->isValid) { - // Remove the job pointer from the labelToInsertJobs mapping. - auto &jobs = labelToInsertJobs.at(job->label); - for (size_t i = 0; i < jobs.size(); i++) { - if (jobs[i]->id == job->id) { - jobs.erase(jobs.begin() + (long)i); - break; - } - } - if (labelToInsertJobs.at(job->label).empty()) { - labelToInsertJobs.erase(job->label); - } - // Remove the vector from the flat buffer. This may cause the last vector id to swap with - // the deleted id. Hold the label for the last id, so we can later on update its - // corresponding job id. Note that after calling deleteVectorById, the last id's label - // shouldn't be available, since it is removed from the lookup. - labelType last_vec_label = - this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); - int deleted = this->frontendIndex->deleteVectorById(job->label, job->id); - if (deleted && job->id != this->frontendIndex->indexSize()) { - // If the vector removal caused a swap with the last id, update the relevant insert job. - this->updateInsertJobInternalId(this->frontendIndex->indexSize(), job->id, - last_vec_label); - } - } else { - // Remove the current job from the invalid jobs' lookup, as we are about to delete it now. - this->invalidJobsLookupGuard.lock(); - this->invalidJobs.erase(job->id); - this->invalidJobsLookupGuard.unlock(); - } - this->flatIndexGuard.unlock(); + this->removeIngestedVectorFromFlat(job); } template @@ -796,9 +715,7 @@ TieredHNSWIndex::TieredHNSWIndex(HNSWIndex allocator) : VecSimTieredIndex(hnsw_index, bf_index, tiered_index_params, allocator), - labelToInsertJobs(this->allocator), idToRepairJobs(this->allocator), - idToSwapJob(this->allocator), invalidJobs(this->allocator), currInvalidJobId(0), - readySwapJobs(0), + idToRepairJobs(this->allocator), idToSwapJob(this->allocator), readySwapJobs(0), isQuantized(tiered_index_params.primaryIndexParams->algoParams.hnswParams.quantType != VecSimQuant_NONE) { const auto &hnsw_params = tiered_index_params.primaryIndexParams->algoParams.hnswParams; @@ -818,12 +735,6 @@ TieredHNSWIndex::TieredHNSWIndex(HNSWIndex TieredHNSWIndex::~TieredHNSWIndex() { - // Delete all the pending insert jobs. - for (auto &jobs : this->labelToInsertJobs) { - for (auto *job : jobs.second) { - delete job; - } - } // Delete all the pending repair jobs. for (auto &jobs : this->idToRepairJobs) { for (auto *job : jobs.second) { @@ -834,10 +745,6 @@ TieredHNSWIndex::~TieredHNSWIndex() { for (auto &it : this->idToSwapJob) { delete it.second; } - // Delete all the pending invalid jobs. - for (auto &it : this->invalidJobs) { - delete it.second; - } } // Training writes stay in FLAT in both write modes, even beyond flatBufferLimit. @@ -848,20 +755,21 @@ int TieredHNSWIndex::addVectorDuringAccumulation(const void const auto [result, count, should_finalize] = [&] { std::lock_guard flat_lock(this->flatIndexGuard); idType id = this->frontendIndex->indexSize(); - HNSWInsertJob *job = nullptr; + TieredInsertJob *job = nullptr; if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { id = static_cast *>(this->frontendIndex) ->getIdOfLabel(label); subtractFromSum( {this->frontendIndex->getDataByInternalId(id), this->frontendIndex->getDim()}); - job = labelToInsertJobs.at(label).front(); + job = this->labelToInsertJobs.at(label).front(); } const int result = this->frontendIndex->addVector(blob, label); addToSum({this->frontendIndex->getDataByInternalId(id), this->frontendIndex->getDim()}); if (!job) { job = new (this->allocator) HNSWInsertJob(this->allocator, label, id, executeInsertJobWrapper, this); - auto [it, inserted] = labelToInsertJobs.try_emplace(label, 1, job, this->allocator); + auto [it, inserted] = + this->labelToInsertJobs.try_emplace(label, 1, job, this->allocator); if (!inserted) { it->second.push_back(job); } @@ -880,8 +788,8 @@ int TieredHNSWIndex::addVectorDuringAccumulation(const void template int TieredHNSWIndex::deleteVectorDuringAccumulation(labelType label) { std::lock_guard flat_lock(this->flatIndexGuard); - auto it = labelToInsertJobs.find(label); - if (it == labelToInsertJobs.end()) { + auto it = this->labelToInsertJobs.find(label); + if (it == this->labelToInsertJobs.end()) { return 0; } const int removed = it->second.size(); @@ -890,10 +798,10 @@ int TieredHNSWIndex::deleteVectorDuringAccumulation(labelTyp {this->frontendIndex->getDataByInternalId(job->id), this->frontendIndex->getDim()}); delete job; // Never submitted, so no worker can still reference this job. } - labelToInsertJobs.erase(it); + this->labelToInsertJobs.erase(it); auto updated_ids = this->frontendIndex->deleteVectorAndGetUpdatedIds(label); for (const auto &[new_id, previous] : updated_ids) { - updateInsertJobInternalId(previous.first, new_id, previous.second); + this->updateInsertJobInternalId(previous.first, new_id, previous.second); } return removed; } @@ -920,7 +828,7 @@ void TieredHNSWIndex::finalizeQuantizationAndSubmitJobs(size jobs.reserve(count); { std::shared_lock flat_lock(this->flatIndexGuard); - for (const auto &[label, pending] : labelToInsertJobs) { + for (const auto &[label, pending] : this->labelToInsertJobs) { jobs.insert(jobs.end(), pending.begin(), pending.end()); } } @@ -1177,7 +1085,7 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o if (jobs_it != this->labelToInsertJobs.end()) { auto jobs = std::move(jobs_it->second); this->labelToInsertJobs.erase(jobs_it); - for (HNSWInsertJob *job : jobs) { + for (TieredInsertJob *job : jobs) { job->label = new_label; } this->labelToInsertJobs.emplace(new_label, std::move(jobs)); diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 4d0ebe2cb..0e49c4820 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -17,14 +17,12 @@ /** * Definition of a job that inserts a new vector from flat into SVS Index. */ -struct SVSInsertJob : public AsyncJob { - labelType label; - idType id; +struct SVSInsertJob : public TieredInsertJob { std::atomic executing{false}; SVSInsertJob(std::shared_ptr allocator, labelType label_, idType id_, JobCallback insertCb, VecSimIndex *index_) - : AsyncJob(allocator, SVS_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} + : TieredInsertJob(allocator, SVS_INSERT_VECTOR_JOB, label_, id_, insertCb, index_) {} }; /** @@ -271,12 +269,6 @@ class TieredSVSIndex : public VecSimTieredIndex { std::atomic backendInitSubmited{false}; std::unordered_set ids_to_init_; - vecsim_stl::unordered_map> labelToInsertJobs; - // A mapping to hold invalid jobs, so we can dispose them upon index deletion. - vecsim_stl::unordered_map invalidJobs; - idType currInvalidJobId; // A unique arbitrary identifier for accessing invalid jobs - std::mutex invalidJobsLookupGuard; - vecsim_stl::unordered_map> labelToConsolidateJobs; mutable std::shared_mutex consolidateJobsGuard; @@ -665,20 +657,6 @@ class TieredSVSIndex : public VecSimTieredIndex { public: #endif - void updateInsertJobInternalId(idType prev_id, idType new_id, labelType label) { - // Update the pending job id, due to a swap that was caused after the removal of new_id. - assert(new_id != INVALID_ID && prev_id != INVALID_ID); - auto it = this->labelToInsertJobs.find(label); - if (it != this->labelToInsertJobs.end()) { - // There is a pending job for the label of the swapped last id - update its id. - for (SVSInsertJob *job_it : it->second) { - if (job_it->id == prev_id) { - job_it->id = new_id; - } - } - } - } - void scheduleSVSIndexInit() { // do not schedule if scheduled already if (indexUpdateScheduled.test_and_set()) { @@ -760,29 +738,23 @@ class TieredSVSIndex : public VecSimTieredIndex { } } for (auto *job : running) { - while (job->executing.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } + waitForJob(job); } return taken_over; } // Wait until the job leaves its publish window // Safe to call while holding flatIndexGuard exclusive - static void waitForInsertJob(SVSInsertJob *job) { + template + static void waitForJob(Job *job) { while (job->executing.load(std::memory_order_acquire)) { std::this_thread::yield(); } } - idType setAndSaveInvalidJob(SVSInsertJob *job) { - waitForInsertJob(job); - this->invalidJobsLookupGuard.lock(); - job->isValid = false; - idType curInvalidId = currInvalidJobId++; - this->invalidJobs.insert({curInvalidId, job}); - this->invalidJobsLookupGuard.unlock(); - return curInvalidId; + idType setAndSaveInvalidJob(AsyncJob *job) override { + waitForJob(static_cast(job)); + return Base::setAndSaveInvalidJob(job); } enum class InsertJobOutcome { Completed, Deferred }; @@ -822,43 +794,7 @@ class TieredSVSIndex : public VecSimTieredIndex { job->executing.store(false, std::memory_order_release); // Remove the vector and the insert job from the flat buffer. - this->flatIndexGuard.lock(); - // The job might have been invalidated due to overwrite in the meantime. In this case, - // it was already deleted and the job has been evicted. Otherwise, we need to do it now. - if (job->isValid) { - // Remove the job pointer from the labelToInsertJobs mapping. - auto &jobs = labelToInsertJobs.at(job->label); - for (size_t i = 0; i < jobs.size(); i++) { - if (jobs[i]->id == job->id) { - jobs.erase(jobs.begin() + (long)i); - break; - } - } - if (labelToInsertJobs.at(job->label).empty()) { - labelToInsertJobs.erase(job->label); - } - // Remove the vector from the flat buffer. - // The flat buffer stores data in a contiguous - // array, so deleting an element may move the last element into the freed slot to keep - // ids dense. Capture the last id's label beforehand (after deletion it is no longer in - // the lookup) so we can later fix up its insert job. - labelType last_vec_label = - this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); - int deleted = this->frontendIndex->deleteVectorById(job->label, job->id); - if (deleted && job->id != this->frontendIndex->indexSize()) { - // If the vector removal caused a swap with the last id, update the relevant insert - // job. - this->updateInsertJobInternalId(this->frontendIndex->indexSize(), job->id, - last_vec_label); - } - } else { - // Remove the current job from the invalid jobs' lookup, as we are about to delete it - // now. - this->invalidJobsLookupGuard.lock(); - this->invalidJobs.erase(job->id); - this->invalidJobsLookupGuard.unlock(); - } - this->flatIndexGuard.unlock(); + this->removeIngestedVectorFromFlat(job); return InsertJobOutcome::Completed; } @@ -936,9 +872,7 @@ class TieredSVSIndex : public VecSimTieredIndex { const TieredIndexParams &tiered_index_params, std::shared_ptr allocator) : Base(svs_index, bf_index, tiered_index_params, allocator), - uncompletedJobs(this->allocator), labelToInsertJobs(this->allocator), - invalidJobs(this->allocator), currInvalidJobId(0), - labelToConsolidateJobs(this->allocator) { + uncompletedJobs(this->allocator), labelToConsolidateJobs(this->allocator) { const auto &tiered_svs_params = tiered_index_params.specificParams.tieredSVSParams; // If flatBufferLimit is not initialized (0), use the default update threshold. @@ -967,18 +901,6 @@ class TieredSVSIndex : public VecSimTieredIndex { } ~TieredSVSIndex() { - // Delete all the pending insert jobs. - for (auto &jobs : this->labelToInsertJobs) { - for (auto *job : jobs.second) { - delete job; - } - } - - // Delete all the pending invalid jobs. - for (auto &it : this->invalidJobs) { - delete it.second; - } - // Delete all the pending consolidate jobs for (auto &jobs : this->labelToConsolidateJobs) { for (auto *job : jobs.second) { @@ -1109,7 +1031,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // If this label already exists, this will do overwrite. ret += this->frontendIndex->addVector(blob, label); - AsyncJob *new_insert_job = new (this->allocator) + TieredInsertJob *new_insert_job = new (this->allocator) SVSInsertJob(this->allocator, label, new_flat_id, executeInsertJobWrapper, this); // Save a pointer to the job, so that if the vector is overwritten, we'll have an @@ -1118,10 +1040,10 @@ class TieredSVSIndex : public VecSimTieredIndex { // There's already a pending insert job for this label, add another one (without // overwrite, only possible in multi index) assert(this->backendIndex->isMultiValue()); - this->labelToInsertJobs.at(label).push_back((SVSInsertJob *)new_insert_job); + this->labelToInsertJobs.at(label).push_back(new_insert_job); } else { - vecsim_stl::vector new_jobs_vec(1, (SVSInsertJob *)new_insert_job, - this->allocator); + vecsim_stl::vector new_jobs_vec(1, new_insert_job, + this->allocator); this->labelToInsertJobs.insert({label, new_jobs_vec}); } this->flatIndexGuard.unlock(); @@ -1277,7 +1199,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto jobs = std::move(pending->second); this->labelToInsertJobs.erase(pending); for (auto *job : jobs) { - waitForInsertJob(job); + waitForJob(static_cast(job)); job->label = new_label; } this->labelToInsertJobs.emplace(new_label, std::move(jobs)); @@ -1312,7 +1234,7 @@ class TieredSVSIndex : public VecSimTieredIndex { auto jobs = std::move(pending->second); this->labelToInsertJobs.erase(pending); for (auto *job : jobs) { - waitForInsertJob(job); + waitForJob(static_cast(job)); job->label = old_label; } this->labelToInsertJobs.emplace(old_label, std::move(jobs)); diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index ce159ec46..84fee4338 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -43,6 +43,19 @@ struct AsyncJob : public VecsimBaseObject { isValid(true) {} }; +/** + * Definition of a job that inserts a new vector from flat into the backend index. + * Backend specific insert jobs derive from it to set their own job type. + */ +struct TieredInsertJob : public AsyncJob { + labelType label; + idType id; + + TieredInsertJob(std::shared_ptr allocator, JobType type, labelType label_, + idType id_, JobCallback insertCb, VecSimIndex *index_) + : AsyncJob(allocator, type, insertCb, index_), label(label_), id(id_) {} +}; + // All read operations (including KNN, range, batch iterators and get-distance-from) are guaranteed // to consider all vectors that were added to the index before the query was submitted. The results // may include vectors that were added after the query was submitted, with no guarantees. @@ -134,6 +147,83 @@ class VecSimTieredIndex : public VecSimIndexInterface { VecSimQueryParams *queryParams, VecSimQueryReply_Order order) const; +#ifdef BUILD_TESTS +public: +#endif + /// Mappings from id/label to associated jobs, for invalidating and update ids if necessary. + // In MULTI, we can have more than one insert job pending per label. + // **This map is protected with the flat buffer lock** + vecsim_stl::unordered_map> labelToInsertJobs; + + // Helper function for updating the pending insert job(s) of a label after the flat buffer + // swapped the vector's internal id + void updateInsertJobInternalId(idType prev_id, idType new_id, labelType label) { + // Update the pending job id, due to a swap that was caused after the removal of new_id. + assert(new_id != INVALID_ID && prev_id != INVALID_ID); + auto it = this->labelToInsertJobs.find(label); + if (it != this->labelToInsertJobs.end()) { + // There is a pending job for the label of the swapped last id - update its id. + for (TieredInsertJob *job_it : it->second) { + if (job_it->id == prev_id) { + job_it->id = new_id; + } + } + } + } + + // A mapping to hold invalid jobs, so we can dispose them upon index deletion. + vecsim_stl::unordered_map invalidJobs; + idType currInvalidJobId; // A unique arbitrary identifier for accessing invalid jobs + std::mutex invalidJobsLookupGuard; + + // Set an insert/repair job as invalid, put the job pointer in the invalid jobs lookup under + // the current available id, increase it and return it (while holding invalidJobsLookupGuard). + // Returns the id that the job was stored under (to be set in the job id field). + virtual idType setAndSaveInvalidJob(AsyncJob *job) { + std::lock_guard lock(this->invalidJobsLookupGuard); + job->isValid = false; + idType curInvalidId = currInvalidJobId++; + this->invalidJobs.insert({curInvalidId, job}); + return curInvalidId; + } + + // Remove a vector and its insert job from the flat buffer + void removeIngestedVectorFromFlat(TieredInsertJob *job) { + std::lock_guard flat_lock(this->flatIndexGuard); + // The job might have been invalidated due to overwrite in the meantime. In this case, + // it was already deleted and the job has been evicted. Otherwise, we need to do it now. + if (!job->isValid) { + // Remove the current job from the invalid jobs' lookup, as we are about to delete it + // now. + std::lock_guard invalid_jobs_lock(this->invalidJobsLookupGuard); + this->invalidJobs.erase(job->id); + return; + } + // Remove the job pointer from the labelToInsertJobs mapping. + auto &jobs = this->labelToInsertJobs.at(job->label); + for (size_t i = 0; i < jobs.size(); i++) { + if (jobs[i]->id == job->id) { + jobs.erase(jobs.begin() + (long)i); + break; + } + } + if (jobs.empty()) { + this->labelToInsertJobs.erase(job->label); + } + // Remove the vector from the flat buffer. This may cause the last vector id to swap with + // the deleted id. Hold the label for the last id, so we can later on update its + // corresponding job id. Note that after calling deleteVectorById, the last id's label + // shouldn't be available, since it is removed from the lookup. + labelType last_vec_label = + this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); + int deleted = this->frontendIndex->deleteVectorById(job->label, job->id); + if (deleted && job->id != this->frontendIndex->indexSize()) { + // If the vector removal caused a swap with the last id, update the relevant insert job. + this->updateInsertJobInternalId(this->frontendIndex->indexSize(), job->id, + last_vec_label); + } + } + public: /** * @brief Get the vector elements stored under a label, in insertion order. @@ -207,11 +297,22 @@ class VecSimTieredIndex : public VecSimIndexInterface { : VecSimIndexInterface(allocator), backendIndex(backendIndex_), frontendIndex(frontendIndex_), jobQueue(tieredParams.jobQueue), jobQueueCtx(tieredParams.jobQueueCtx), SubmitJobsToQueue(tieredParams.submitCb), - flatBufferLimit(tieredParams.flatBufferLimit) { + flatBufferLimit(tieredParams.flatBufferLimit), labelToInsertJobs(this->allocator), + invalidJobs(this->allocator), currInvalidJobId(0) { assert(backendIndex != nullptr); } virtual ~VecSimTieredIndex() { + // Delete all the pending insert jobs. + for (auto &jobs : this->labelToInsertJobs) { + for (auto *job : jobs.second) { + delete job; + } + } + // Delete all the pending invalid jobs. + for (auto &it : this->invalidJobs) { + delete it.second; + } VecSimIndex_Free(backendIndex); VecSimIndex_Free(frontendIndex); } diff --git a/tests/unit/test_hnsw_tiered.cpp b/tests/unit/test_hnsw_tiered.cpp index 9b3600cdf..29702c24c 100644 --- a/tests/unit/test_hnsw_tiered.cpp +++ b/tests/unit/test_hnsw_tiered.cpp @@ -124,7 +124,7 @@ TYPED_TEST(HNSWTieredIndexTest, CreateIndexInstance) { auto job = new (allocator) HNSWInsertJob(tiered_index->allocator, vector_label, 0, insert_to_index, tiered_index); - auto jobs_vec = vecsim_stl::vector(1, job, allocator); + auto jobs_vec = vecsim_stl::vector(1, job, allocator); tiered_index->labelToInsertJobs.insert({vector_label, jobs_vec}); // Wrap this job with an array and submit the jobs to the queue. @@ -339,10 +339,10 @@ TYPED_TEST(HNSWTieredIndexTest, addVector) { sizeof(void *) + sizeof(size_t); // Account for the memory that was allocated in the labelToInsertJobs map (approx.) expected_mem += - sizeof( - vecsim_stl::unordered_map>::value_type) + + sizeof(vecsim_stl::unordered_map>::value_type) + sizeof(void *) + sizeof(size_t); - // Account for the inner buffer of the std::vector in the map. + // Account for the inner buffer of the std::vector in the map. expected_mem += sizeof(void *) + sizeof(size_t); // Account for the insert job that was created. expected_mem += sizeof(HNSWInsertJob) + sizeof(size_t); @@ -4988,7 +4988,7 @@ TYPED_TEST(HNSWTieredIndexTestBasic, relabelVectorMulti) { } ASSERT_EQ(tiered_index->labelToInsertJobs.count(7), 0); ASSERT_EQ(tiered_index->labelToInsertJobs.at(70).size(), per_label); - for (HNSWInsertJob *job : tiered_index->labelToInsertJobs.at(70)) { + for (TieredInsertJob *job : tiered_index->labelToInsertJobs.at(70)) { ASSERT_EQ(job->label, 70); } From b48755cee60a432e1dc8b415a388500c944717f2 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 01:32:18 -0700 Subject: [PATCH 53/64] format --- tests/unit/test_hnsw_tiered.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_hnsw_tiered.cpp b/tests/unit/test_hnsw_tiered.cpp index 29702c24c..46afdede6 100644 --- a/tests/unit/test_hnsw_tiered.cpp +++ b/tests/unit/test_hnsw_tiered.cpp @@ -340,7 +340,7 @@ TYPED_TEST(HNSWTieredIndexTest, addVector) { // Account for the memory that was allocated in the labelToInsertJobs map (approx.) expected_mem += sizeof(vecsim_stl::unordered_map>::value_type) + + vecsim_stl::vector>::value_type) + sizeof(void *) + sizeof(size_t); // Account for the inner buffer of the std::vector in the map. expected_mem += sizeof(void *) + sizeof(size_t); From 9e602e90ca3df95806614633f2b23c16deac747d Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Wed, 16 Sep 2026 01:42:01 -0700 Subject: [PATCH 54/64] upstream getDistanceFrom_Unsafe --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 53 ++---------------------- src/VecSim/algorithms/svs/svs_tiered.h | 20 --------- src/VecSim/vec_sim_tiered_index.h | 45 ++++++++++++++++++++ 3 files changed, 49 insertions(+), 69 deletions(-) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index 5a7760875..9180b91ff 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -288,7 +288,6 @@ class TieredHNSWIndex : public VecSimTieredIndex { int deleteVector(labelType label) override; VecSimRelabelCode relabelVector(labelType old_label, labelType new_label) override; size_t getNumMarkedDeleted() const override { return getHNSWIndex()->getNumMarkedDeleted(); } - double getDistanceFrom_Unsafe(labelType label, const void *blob) const override; // Do nothing here, each tier (flat buffer and HNSW) should increase capacity for itself when // needed. VecSimIndexDebugInfo debugInfo() const override; @@ -342,16 +341,16 @@ class TieredHNSWIndex : public VecSimTieredIndex { /* Helper methods */ template void TieredHNSWIndex::executeInsertJobWrapper(AsyncJob *job) { - auto *insert_job = reinterpret_cast(job); - auto *job_index = reinterpret_cast *>(insert_job->index); + auto *insert_job = static_cast(job); + auto *job_index = static_cast *>(insert_job->index); job_index->executeInsertJob(insert_job); delete job; } template void TieredHNSWIndex::executeRepairJobWrapper(AsyncJob *job) { - auto *repair_job = reinterpret_cast(job); - auto *job_index = reinterpret_cast *>(repair_job->index); + auto *repair_job = static_cast(job); + auto *job_index = static_cast *>(repair_job->index); job_index->executeRepairJob(repair_job); delete job; } @@ -1116,50 +1115,6 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o return VecSimRelabel_OK; } -// `getDistanceFrom` returns the minimum distance between the given blob and the vector with the -// given label. If the label doesn't exist, the distance will be NaN. -// Therefore, it's better to just call `getDistanceFrom` on both indexes and return the minimum -// instead of checking if the label exists in each index. We first try to get the distance from the -// flat buffer, as vectors in the buffer might move to the Main while we're "between" the locks. -// Behavior for single (regular) index: -// 1. label doesn't exist in both indexes - return NaN -// 2. label exists in one of the indexes only - return the distance from that index (which is valid) -// 3. label exists in both indexes - return the value from the flat buffer (which is valid and equal -// to the value from the Main index), saving us from locking the Main index. -// Behavior for multi index: -// 1. label doesn't exist in both indexes - return NaN -// 2. label exists in one of the indexes only - return the distance from that index (which is valid) -// 3. label exists in both indexes - we may have some of the vectors with the same label in the flat -// buffer only and some in the Main index only (and maybe temporal duplications). -// So, we get the distance from both indexes and return the minimum. - -// IMPORTANT: this should be called when the *tiered index locks are locked for shared ownership*, -// along with HNSW index data guard lock. That is since the internal getDistanceFrom calls access -// the indexes' data, and it is not safe to run insert/delete operation in parallel. Also, we avoid -// acquiring the locks internally, since this is usually called for every vector individually, and -// the overhead of acquiring and releasing the locks is significant in that case. -template -double TieredHNSWIndex::getDistanceFrom_Unsafe(labelType label, - const void *blob) const { - // Try to get the distance from the flat buffer. - // If the label doesn't exist, the distance will be NaN. - auto flat_dist = this->frontendIndex->getDistanceFrom_Unsafe(label, blob); - - // Optimization. TODO: consider having different implementations for single and multi indexes, - // to avoid checking the index type on every query. - if (!this->backendIndex->isMultiValue() && !std::isnan(flat_dist)) { - // If the index is single value, and we got a valid distance from the flat buffer, - // we can return the distance without querying the Main index. - return flat_dist; - } - - // Try to get the distance from the Main index. - auto hnsw_dist = getHNSWIndex()->getDistanceFrom_Unsafe(label, blob); - - // Return the minimum distance that is not NaN. - return std::fmin(flat_dist, hnsw_dist); -} - //////////////////////////////////////////////////////////////////////////////////////////////////// // TieredHNSW_BatchIterator // //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 0e49c4820..67955d724 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -1253,26 +1253,6 @@ class TieredSVSIndex : public VecSimTieredIndex { return this->GetSVSIndex()->getNumMarkedDeleted(); } - double getDistanceFrom_Unsafe(labelType label, const void *blob) const override { - // Try to get the distance from the flat buffer. - // If the label doesn't exist, the distance will be NaN. - auto flat_dist = this->frontendIndex->getDistanceFrom_Unsafe(label, blob); - - // Optimization. TODO: consider having different implementations for single and multi - // indexes, to avoid checking the index type on every query. - if (!this->backendIndex->isMultiValue() && !std::isnan(flat_dist)) { - // If the index is single value, and we got a valid distance from the flat buffer, - // we can return the distance without querying the Main index. - return flat_dist; - } - - // Try to get the distance from the Main index. - auto svs_dist = this->backendIndex->getDistanceFrom_Unsafe(label, blob); - - // Return the minimum distance that is not NaN. - return std::fmin(flat_dist, svs_dist); - } - VecSimIndexDebugInfo debugInfo() const override { auto info = Base::debugInfo(); diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 84fee4338..d28b7c9f2 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -18,6 +18,7 @@ #include "VecSim/utils/scoped_locks.h" #include +#include #include #include @@ -291,6 +292,50 @@ class VecSimTieredIndex : public VecSimIndexInterface { } } + // `getDistanceFrom` returns the minimum distance between the given blob and the vector with + // the given label. If the label doesn't exist, the distance will be NaN. + // Therefore, it's better to just call `getDistanceFrom` on both indexes and return the minimum + // instead of checking if the label exists in each index. We first try to get the distance from + // the flat buffer, as vectors in the buffer might move to the backend while we're "between" + // the locks. + // Behavior for single (regular) index: + // 1. label doesn't exist in both indexes - return NaN + // 2. label exists in one of the indexes only - return the distance from that index (valid) + // 3. label exists in both indexes - return the value from the flat buffer (valid and equal to + // the value from the backend index), saving us from locking the backend index. + // Behavior for multi index: + // 1. label doesn't exist in both indexes - return NaN + // 2. label exists in one of the indexes only - return the distance from that index (valid) + // 3. label exists in both indexes - we may have some of the vectors with the same label in the + // flat buffer only and some in the backend index only (and maybe temporal duplications). So, + // we get the distance from both indexes and return the minimum. + // + // IMPORTANT: this should be called when the *tiered index locks are locked for shared + // ownership*, along with the backend index's own data guard lock if it has one. That is since + // the internal getDistanceFrom calls access the indexes' data, and it is not safe to run + // insert/delete operations in parallel. Also, we avoid acquiring the locks internally, since + // this is usually called for every vector individually, and the overhead of acquiring and + // releasing the locks is significant in that case. + double getDistanceFrom_Unsafe(labelType label, const void *blob) const override { + // Try to get the distance from the flat buffer. + // If the label doesn't exist, the distance will be NaN. + auto flat_dist = this->frontendIndex->getDistanceFrom_Unsafe(label, blob); + + // Optimization. TODO: consider having different implementations for single and multi + // indexes, to avoid checking the index type on every query. + if (!this->backendIndex->isMultiValue() && !std::isnan(flat_dist)) { + // If the index is single value, and we got a valid distance from the flat buffer, + // we can return the distance without querying the backend index. + return flat_dist; + } + + // Try to get the distance from the backend index. + auto backend_dist = this->backendIndex->getDistanceFrom_Unsafe(label, blob); + + // Return the minimum distance that is not NaN. + return std::fmin(flat_dist, backend_dist); + } + VecSimTieredIndex(VecSimIndexAbstract *backendIndex_, BruteForceIndex *frontendIndex_, TieredIndexParams tieredParams, std::shared_ptr allocator) From c5c24184720b3ffd1a5ba37124d9e9e88e28d7e2 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 04:50:43 -0700 Subject: [PATCH 55/64] fix Full-buffer overwrite skips pending jobs --- src/VecSim/algorithms/svs/svs_tiered.h | 75 +++++++++++++------------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 0e49c4820..04e78a3d1 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -990,14 +990,12 @@ class TieredSVSIndex : public VecSimTieredIndex { this->flatIndexGuard.unlock_shared(); auto storage_blob = this->frontendIndex->preprocessForStorage(blob); - if (!this->backendIndex->isMultiValue()) { - auto deleted = svs_index->deleteVector(label); - if (deleted > 0) - scheduleSVSIndexConsolidate(label); - } + const int overwritten = + this->backendIndex->isMultiValue() ? 0 : this->deleteVector(label); std::shared_lock lock(updateJobMutex); - return ret = svs_index->addVector(storage_blob.get(), label); + ret = svs_index->addVector(storage_blob.get(), label); + return std::max(ret - overwritten, 0); } else { this->flatIndexGuard.unlock_shared(); this->flatIndexGuard.lock(); @@ -1101,6 +1099,40 @@ class TieredSVSIndex : public VecSimTieredIndex { } } + int removeLabelFromFlat(labelType label) { + if (!this->frontendIndex->isLabelExists(label)) { + return 0; + } + auto deleting_ids = this->frontendIndex->getElementIds(label); + if (deleting_ids.size() == 0) { + return 0; + } + + // assert if all elements of deleting_ids are unique + assert(std::set(deleting_ids.begin(), deleting_ids.end()).size() == deleting_ids.size() && + "deleting_ids should contain unique ids"); + + // If id is deleted, don't use it for initialization + if (!ids_to_init_.empty()) { + for (idType id : deleting_ids) { + ids_to_init_.erase(id); + } + } + + if (this->labelToInsertJobs.count(label) > 0) { + // Invalidate the pending insert job(s) into SVS associated with this label + auto &insert_jobs = this->labelToInsertJobs.at(label); + for (auto *job : insert_jobs) { + job->id = this->setAndSaveInvalidJob(job); + } + // Remove the pending insert job(s) from the labelToInsertJobs mapping. + this->labelToInsertJobs.erase(label); + } + + deleteAndUpdateInitIds(label); + return static_cast(deleting_ids.size()); + } + int deleteVector(labelType label) override { int ret = 0; @@ -1108,36 +1140,7 @@ class TieredSVSIndex : public VecSimTieredIndex { if (this->frontendIndex->isLabelExists(label)) { this->flatIndexGuard.unlock_shared(); std::lock_guard flat_lock{this->flatIndexGuard}; - // Check again if the label exists, as it may have been removed while we released the - // lock. - if (this->frontendIndex->isLabelExists(label)) { - auto deleting_ids = this->frontendIndex->getElementIds(label); - if (deleting_ids.size() == 0) - return 0; - - // assert if all elements of deleting_ids are unique - assert(std::set(deleting_ids.begin(), deleting_ids.end()).size() == - deleting_ids.size() && - "deleting_ids should contain unique ids"); - - // If id is deleted, don't use it for initialization - for (idType id : deleting_ids) { - ids_to_init_.erase(id); - } - - if (this->labelToInsertJobs.count(label) > 0) { - // Invalidate the pending insert job(s) into SVS associated with this label - auto &insert_jobs = this->labelToInsertJobs.at(label); - for (auto *job : insert_jobs) { - job->id = this->setAndSaveInvalidJob(job); - } - // Remove the pending insert job(s) from the labelToInsertJobs mapping. - this->labelToInsertJobs.erase(label); - } - - deleteAndUpdateInitIds(label); - ret += deleting_ids.size(); - } + ret += removeLabelFromFlat(label); } else { this->flatIndexGuard.unlock_shared(); } From b2873d8ef1ea3b2dbdc572783c931fbbd2bb4c73 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 05:00:19 -0700 Subject: [PATCH 56/64] fix Init overwrite reports a new insert --- src/VecSim/algorithms/svs/svs_tiered.h | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 04e78a3d1..ceec1a33e 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -925,12 +925,13 @@ class TieredSVSIndex : public VecSimTieredIndex { // initSVSIndexWrapper() is called. { std::lock_guard lock(this->flatIndexGuard); + int deleted = 0; if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { - deleteAndUpdateInitIds(label); + deleted = deleteAndUpdateInitIds(label); } ids_to_init_.insert(this->frontendIndex->indexSize()); - ret = this->frontendIndex->addVector(blob, label); + ret = std::max(this->frontendIndex->addVector(blob, label) - deleted, 0); // If frontend size exceeds the update job threshold, ... frontend_index_size = this->frontendIndex->indexSize(); } @@ -969,13 +970,14 @@ class TieredSVSIndex : public VecSimTieredIndex { std::lock_guard lock(this->flatIndexGuard); if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { + int deleted = 0; if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { - deleteAndUpdateInitIds(label); + deleted = deleteAndUpdateInitIds(label); } ids_to_init_.insert(this->frontendIndex->indexSize()); const auto ft_ret = this->frontendIndex->addVector(blob, label); - ret = std::max(ret + ft_ret, 0); + ret = std::max(ret + ft_ret - deleted, 0); if (this->frontendIndex->indexSize() >= this->trainingTriggerThreshold) { this->backendInitSubmited.store(true, std::memory_order_release); @@ -1066,7 +1068,8 @@ class TieredSVSIndex : public VecSimTieredIndex { } } - void deleteAndUpdateInitIds(labelType label) { + // Returns the number of vectors removed from the frontend index. + int deleteAndUpdateInitIds(labelType label) { auto deleting_ids = this->frontendIndex->getElementIds(label); std::sort(deleting_ids.begin(), deleting_ids.end()); @@ -1097,6 +1100,8 @@ class TieredSVSIndex : public VecSimTieredIndex { for (idType new_id : new_ids_to_init) { ids_to_init_.insert(new_id); } + + return static_cast(deleting_ids.size()); } int removeLabelFromFlat(labelType label) { From 2089fedf5172f07e92237ef0b6cedae4337b3096 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 06:28:02 -0700 Subject: [PATCH 57/64] fix Empty index skips compression retraining --- src/VecSim/algorithms/svs/svs_tiered.h | 163 ++++++++++++++++--------- src/VecSim/vec_sim_tiered_index.h | 32 +++-- 2 files changed, 125 insertions(+), 70 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index ceec1a33e..d11b3ea29 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -266,7 +267,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // The reason of following container just to properly destroy jobs which not executed yet SVSMultiThreadJob::JobsRegistry uncompletedJobs; - std::atomic backendInitSubmited{false}; + // frontend ids holding training data for the backend (re)initialization std::unordered_set ids_to_init_; vecsim_stl::unordered_map> @@ -550,9 +551,9 @@ class TieredSVSIndex : public VecSimTieredIndex { * * This static wrapper function performs the following actions: * - Acquires a lock on the index's updateJobMutex to prevent concurrent updates. - * - Clears the indexUpdateScheduled flag to allow future scheduling. * - Configures the number of threads for the underlying SVS index update operation. * - Calls the initSVSIndex method to perform the actual index update. + * - Clears the indexUpdateScheduled flag to allow future scheduling. * * @param idx Pointer to the VecSimIndex to be updated. * @param availableThreads The number of threads available for the update operation. Current @@ -564,9 +565,11 @@ class TieredSVSIndex : public VecSimTieredIndex { assert(index); // prevent parallel updates std::lock_guard lock(index->updateJobMutex); - // Release the scheduled flag to allow scheduling again - index->indexUpdateScheduled.clear(); - // Update the SVS index + // flag stays set while init runs: !ready() + !flag means "no one will init the backend" + struct ClearOnExit { + std::atomic_flag &flag; + ~ClearOnExit() { flag.clear(std::memory_order_release); } + } clear_on_exit{index->indexUpdateScheduled}; index->initSVSIndex(availableThreads); } @@ -775,10 +778,13 @@ class TieredSVSIndex : public VecSimTieredIndex { return InsertJobOutcome::Completed; } - // Backend is still initilizing. Resubmit the job - if (!svs_index->ready() && this->indexUpdateScheduled.test()) { + // a job never inits the backend: one point cannot train the compression + if (!svs_index->ready()) { this->flatIndexGuard.unlock_shared(); - return InsertJobOutcome::Deferred; + // init pending -> wait for it. no init pending -> back to the training buffer + return this->indexUpdateScheduled.test(std::memory_order_acquire) + ? InsertJobOutcome::Deferred + : adoptJobIntoInitBuffer(job); } // Copy the vector blob out of the flat buffer while holding flatIndexGuard, so we @@ -912,39 +918,20 @@ class TieredSVSIndex : public VecSimTieredIndex { int addVector(const void *blob, labelType label) override { int ret = 0; auto svs_index = GetSVSIndex(); - size_t frontend_index_size = 0; // In-Place mode - add vector syncronously to the backend index. if (this->getWriteMode() == VecSim_WriteInPlace) { // Backend index initialization data have to be buffered for proper // compression/training. - if ((!svs_index->ready()) && - (!this->backendInitSubmited.load(std::memory_order_acquire))) { - // First collect vectors in frontend index - // lock in scope to ensure that these will be released before - // initSVSIndexWrapper() is called. - { - std::lock_guard lock(this->flatIndexGuard); - int deleted = 0; - if (!this->frontendIndex->isMultiValue() && - this->frontendIndex->isLabelExists(label)) { - deleted = deleteAndUpdateInitIds(label); - } - ids_to_init_.insert(this->frontendIndex->indexSize()); - ret = std::max(this->frontendIndex->addVector(blob, label) - deleted, 0); - // If frontend size exceeds the update job threshold, ... - frontend_index_size = this->frontendIndex->indexSize(); - } - // ... move vectors to the backend index. - if (frontend_index_size >= this->trainingTriggerThreshold) { - // initSVSIndexWrapper() accures it's own locks - // initialize the SVS index synchonously using current thread only - initSVSIndexWrapper(this, 1); + if (!svs_index->ready()) { + if (auto buffered = tryBufferForTraining(blob, label, /*in_place=*/true)) { + return *buffered; } - return ret; - } else { - // backend index is initialized - we can add the vector directly - auto storage_blob = this->frontendIndex->preprocessForStorage(blob); + } + // backend index is initialized - we can add the vector directly + auto storage_blob = this->frontendIndex->preprocessForStorage(blob); + int deleted = 0; + { // prevent update job from running in parallel and lock any access to the backend // index // Only updateJobMutex is needed here, not mainIndexGuard: the concurrent @@ -953,37 +940,30 @@ class TieredSVSIndex : public VecSimTieredIndex { // Defensive: ensure single-threaded operation for write-in-place mode. // parallelism_ defaults to 1, so this is a no-op in the normal case. svs_index->setParallelism(1); - int deleted = 0; if (!this->backendIndex->isMultiValue()) { deleted = svs_index->deleteVector(label); if (deleted > 0) svs_index->consolidate({label}); } - return this->backendIndex->addVector(storage_blob.get(), label) - deleted; + if (svs_index->ready()) { + return this->backendIndex->addVector(storage_blob.get(), label) - deleted; + } } + // the delete above dropped the last backend vector: adding now would refit the + // compression to this single point + if (auto buffered = tryBufferForTraining(blob, label, /*in_place=*/true)) { + return std::max(*buffered - deleted, 0); + } + // a queued init job rebuilt the backend meanwhile + std::lock_guard lock(this->updateJobMutex); + return std::max(this->backendIndex->addVector(storage_blob.get(), label) - deleted, 0); } assert(this->getWriteMode() != VecSim_WriteInPlace && "InPlace mode returns early"); - // Async mode - add vector to the frontend index and schedule an update job if needed. - if ((!svs_index->ready()) && (!this->backendInitSubmited.load(std::memory_order_acquire))) { - // Add vector to the frontend index. - std::lock_guard lock(this->flatIndexGuard); - if ((!svs_index->ready()) && - (!this->backendInitSubmited.load(std::memory_order_acquire))) { - int deleted = 0; - if (!this->frontendIndex->isMultiValue() && - this->frontendIndex->isLabelExists(label)) { - deleted = deleteAndUpdateInitIds(label); - } - ids_to_init_.insert(this->frontendIndex->indexSize()); - const auto ft_ret = this->frontendIndex->addVector(blob, label); - ret = std::max(ret + ft_ret - deleted, 0); - - if (this->frontendIndex->indexSize() >= this->trainingTriggerThreshold) { - this->backendInitSubmited.store(true, std::memory_order_release); - scheduleSVSIndexInit(); - } - return ret; + // Async mode - buffer training data until the backend can be inited from a batch. + if (!svs_index->ready() && !this->indexUpdateScheduled.test(std::memory_order_acquire)) { + if (auto buffered = tryBufferForTraining(blob, label, /*in_place=*/false)) { + return *buffered; } } @@ -995,6 +975,13 @@ class TieredSVSIndex : public VecSimTieredIndex { const int overwritten = this->backendIndex->isMultiValue() ? 0 : this->deleteVector(label); + // the delete above may have dropped the last backend vector + if (!svs_index->ready()) { + if (auto buffered = tryBufferForTraining(blob, label, /*in_place=*/false)) { + return std::max(*buffered - overwritten, 0); + } + } + std::shared_lock lock(updateJobMutex); ret = svs_index->addVector(storage_blob.get(), label); return std::max(ret - overwritten, 0); @@ -1068,6 +1055,14 @@ class TieredSVSIndex : public VecSimTieredIndex { } } + // training data is tracked by frontend id, so it has to follow the same swaps as insert jobs + void updateInsertJobInternalId(idType prev_id, idType new_id, labelType label) override { + if (ids_to_init_.erase(prev_id) > 0) { + ids_to_init_.insert(new_id); + } + Base::updateInsertJobInternalId(prev_id, new_id, label); + } + // Returns the number of vectors removed from the frontend index. int deleteAndUpdateInitIds(labelType label) { auto deleting_ids = this->frontendIndex->getElementIds(label); @@ -1138,6 +1133,58 @@ class TieredSVSIndex : public VecSimTieredIndex { return static_cast(deleting_ids.size()); } + // Buffer a vector in the frontend index as training data for the backend (re)initialization. + // Returns 1 for a new label, 0 for an overwrite, or nullopt if the backend turned ready while + // the guard was taken + std::optional tryBufferForTraining(const void *blob, labelType label, bool in_place) { + int ret = 0; + bool run_init = false; + { + std::lock_guard flat_lock{this->flatIndexGuard}; + // never buffer into a ready backend: ids_to_init_ would have no consumer + if (GetSVSIndex()->ready()) { + return std::nullopt; + } + int deleted = 0; + if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { + // once the backend has been dropped the buffer may also hold job-backed vectors, + // so the overwrite has to invalidate pending jobs + deleted = removeLabelFromFlat(label); + } + ids_to_init_.insert(this->frontendIndex->indexSize()); + ret = std::max(this->frontendIndex->addVector(blob, label) - deleted, 0); + run_init = ids_to_init_.size() >= this->trainingTriggerThreshold; + if (run_init && !in_place) { + scheduleSVSIndexInit(); + run_init = false; + } + } + if (run_init) { + initSVSIndexWrapper(this, 1); + } + return ret; + } + + // Move a pending job's vector from the insert-job path to the training batch. The vector stays + // in the frontend index, only its ownership changes. + InsertJobOutcome adoptJobIntoInitBuffer(SVSInsertJob *job) { + std::lock_guard flat_lock{this->flatIndexGuard}; + if (!job->isValid) { + std::lock_guard invalid_lock{this->invalidJobsLookupGuard}; + this->invalidJobs.erase(job->id); + return InsertJobOutcome::Completed; + } + if (GetSVSIndex()->ready() || this->indexUpdateScheduled.test(std::memory_order_acquire)) { + return InsertJobOutcome::Deferred; + } + ids_to_init_.insert(job->id); + this->detachInsertJob(job); + if (ids_to_init_.size() >= this->trainingTriggerThreshold) { + scheduleSVSIndexInit(); + } + return InsertJobOutcome::Completed; + } + int deleteVector(labelType label) override { int ret = 0; @@ -1292,7 +1339,7 @@ class TieredSVSIndex : public VecSimTieredIndex { svsTieredInfo.indexUpdateScheduled = (info.tieredInfo.frontendCommonInfo.indexSize > 0) && - this->backendInitSubmited.load(std::memory_order_acquire); + this->indexUpdateScheduled.test(std::memory_order_acquire); info.tieredInfo.specificTieredBackendInfo.svsTieredInfo = svsTieredInfo; // Background indexing is in progress whenever the flat buffer is non-empty diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 84fee4338..1dd5d99e2 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -157,7 +157,7 @@ class VecSimTieredIndex : public VecSimIndexInterface { // Helper function for updating the pending insert job(s) of a label after the flat buffer // swapped the vector's internal id - void updateInsertJobInternalId(idType prev_id, idType new_id, labelType label) { + virtual void updateInsertJobInternalId(idType prev_id, idType new_id, labelType label) { // Update the pending job id, due to a swap that was caused after the removal of new_id. assert(new_id != INVALID_ID && prev_id != INVALID_ID); auto it = this->labelToInsertJobs.find(label); @@ -187,6 +187,24 @@ class VecSimTieredIndex : public VecSimIndexInterface { return curInvalidId; } + // Remove the job pointer from the labelToInsertJobs mapping. Must hold flatIndexGuard. + void detachInsertJob(TieredInsertJob *job) { + auto it = this->labelToInsertJobs.find(job->label); + if (it == this->labelToInsertJobs.end()) { + return; + } + auto &jobs = it->second; + for (size_t i = 0; i < jobs.size(); i++) { + if (jobs[i] == job) { + jobs.erase(jobs.begin() + (long)i); + break; + } + } + if (jobs.empty()) { + this->labelToInsertJobs.erase(it); + } + } + // Remove a vector and its insert job from the flat buffer void removeIngestedVectorFromFlat(TieredInsertJob *job) { std::lock_guard flat_lock(this->flatIndexGuard); @@ -199,17 +217,7 @@ class VecSimTieredIndex : public VecSimIndexInterface { this->invalidJobs.erase(job->id); return; } - // Remove the job pointer from the labelToInsertJobs mapping. - auto &jobs = this->labelToInsertJobs.at(job->label); - for (size_t i = 0; i < jobs.size(); i++) { - if (jobs[i]->id == job->id) { - jobs.erase(jobs.begin() + (long)i); - break; - } - } - if (jobs.empty()) { - this->labelToInsertJobs.erase(job->label); - } + this->detachInsertJob(job); // Remove the vector from the flat buffer. This may cause the last vector id to swap with // the deleted id. Hold the label for the last id, so we can later on update its // corresponding job id. Note that after calling deleteVectorById, the last id's label From 966067ca5d79313c9d6ed963ccf24ea6d919f12a Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 06:40:56 -0700 Subject: [PATCH 58/64] update comment for GC compacts without consolidating deletes --- src/VecSim/algorithms/svs/svs.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 3a6989ae5..917eeba8f 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -831,8 +831,8 @@ class SVSIndex : public VecSimIndexAbstract, fl void runGC() override { if (auto impl = getImpl()) { - // There is documentation for compact(): - // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.compact + // concurrent compact() consolidates every soft-deleted points first + // pending per-label consolidate jobs need not be drained - they become no-ops impl->compact(); } num_marked_deleted.store(0, std::memory_order_relaxed); From 6e19eb6e153cdb73c5f3d7f8cdc4364a23cd0fa9 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 07:39:33 -0700 Subject: [PATCH 59/64] fix Insert job can reinit dropped backend --- src/VecSim/algorithms/svs/svs.h | 19 ++++++++++++++++++- src/VecSim/algorithms/svs/svs_tiered.h | 8 +++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 917eeba8f..0aaba6aec 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -40,8 +40,14 @@ struct SVSIndexBase { SVSIndexBase() : num_marked_deleted{0} {}; virtual ~SVSIndexBase() = default; + // Returned by addVectorsIfInitialized() when nothing was added. + static constexpr int kNotInitialized = -1; + virtual int addVector(const void *vector_data, labelType label) = 0; virtual int addVectors(const void *vectors_data, const labelType *labels, size_t n) = 0; + // Add vectors into an already initialized instance. + virtual int addVectorsIfInitialized(const void *vectors_data, const labelType *labels, + size_t n) = 0; virtual int deleteVector(labelType label) = 0; virtual int deleteVectors(const labelType *labels, size_t n) = 0; virtual void consolidate(const std::vector &labels) = 0; @@ -283,7 +289,8 @@ class SVSIndex : public VecSimIndexAbstract, fl // for the operation. // Important NOTE: For single vector operations (n=1), parallelism should be 1. // For bulk operations (n>1), parallelism should reflect the number of available threads. - int addVectorsImpl(const void *vectors_data, const labelType *labels, size_t n) { + int addVectorsImpl(const void *vectors_data, const labelType *labels, size_t n, + bool allow_init = true) { if (n == 0) { return 0; } @@ -311,6 +318,9 @@ class SVSIndex : public VecSimIndexAbstract, fl this->implMutationGuard_.lock(); if (auto existing = getImpl()) { existing->add_points(points, ids, /*reuse_empty*/ false); + } else if (!allow_init) { + this->implMutationGuard_.unlock(); + return kNotInitialized; } else { auto built = initImpl(points, ids); assert(built != nullptr); @@ -595,6 +605,13 @@ class SVSIndex : public VecSimIndexAbstract, fl return addVectorsImpl(vectors_data, labels, n); } + int addVectorsIfInitialized(const void *vectors_data, const labelType *labels, + size_t n) override { + assert(!(n == 1 && getParallelism() > 1) && + "Can't use more than one thread to insert a single vector"); + return addVectorsImpl(vectors_data, labels, n, /*allow_init=*/false); + } + int deleteVector(labelType label) override { return deleteVectorImpl(label); } int deleteVectors(const labelType *labels, size_t n) override { diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 99e624463..48db63876 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -796,9 +796,15 @@ class TieredSVSIndex : public VecSimTieredIndex { job->executing.store(true, std::memory_order_release); this->flatIndexGuard.unlock_shared(); - svs_index->addVector(blob_copy.get(), job->label); + // if a concurrent deletion drops the instance + // initializing it from this single point would refit the compression + const int added = + svs_index->addVectorsIfInitialized(blob_copy.get(), &job->label, 1); job->executing.store(false, std::memory_order_release); + if (added == SVSIndexBase::kNotInitialized) { + return adoptJobIntoInitBuffer(job); + } // Remove the vector and the insert job from the flat buffer. this->removeIngestedVectorFromFlat(job); return InsertJobOutcome::Completed; From 6479bd35fde02101e3e220553f97bd0b3cce04f4 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 07:52:36 -0700 Subject: [PATCH 60/64] modify comments --- src/VecSim/algorithms/svs/svs.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 0aaba6aec..dfddb7dc0 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -102,6 +102,8 @@ class SVSIndex : public VecSimIndexAbstract, fl using graph_builder_t = SVSGraphBuilder; using graph_type = typename graph_builder_t::graph_type; + // Note. svs::concurrent::* classes differ from the svs::index::vamana::* classes + // in their mutation contracts. using impl_type = std::conditional_t< isMulti, svs::concurrent::MultiMutableVamanaIndex, @@ -334,8 +336,6 @@ class SVSIndex : public VecSimIndexAbstract, fl } void consolidate(const std::vector &labels) override { - // There is documentation for consolidate(): - // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.consolidate if (auto impl = getImpl()) { impl->consolidate(labels); } From 45005b6484fa987e3f9e101545fb8121aca570e7 Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 08:58:15 -0700 Subject: [PATCH 61/64] fix Relabel races pending multi-value ingest --- src/VecSim/algorithms/svs/svs_tiered.h | 82 ++++++++++++++------------ 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 48db63876..376cda726 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -1024,21 +1024,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // If this label already exists, this will do overwrite. ret += this->frontendIndex->addVector(blob, label); - TieredInsertJob *new_insert_job = new (this->allocator) - SVSInsertJob(this->allocator, label, new_flat_id, executeInsertJobWrapper, this); - - // Save a pointer to the job, so that if the vector is overwritten, we'll have an - // indication. - if (this->labelToInsertJobs.find(label) != this->labelToInsertJobs.end()) { - // There's already a pending insert job for this label, add another one (without - // overwrite, only possible in multi index) - assert(this->backendIndex->isMultiValue()); - this->labelToInsertJobs.at(label).push_back(new_insert_job); - } else { - vecsim_stl::vector new_jobs_vec(1, new_insert_job, - this->allocator); - this->labelToInsertJobs.insert({label, new_jobs_vec}); - } + TieredInsertJob *new_insert_job = createInsertJob(label, new_flat_id); this->flatIndexGuard.unlock(); // Here, a worker might ingest the previous vector that was stored under "label" @@ -1139,6 +1125,24 @@ class TieredSVSIndex : public VecSimTieredIndex { return static_cast(deleting_ids.size()); } + // Create a pending insert job for a vector already buffered in the frontend index + // Caller must hold flatIndexGuard exclusively. + TieredInsertJob *createInsertJob(labelType label, idType flat_id) { + TieredInsertJob *job = new (this->allocator) + SVSInsertJob(this->allocator, label, flat_id, executeInsertJobWrapper, this); + auto it = this->labelToInsertJobs.find(label); + if (it != this->labelToInsertJobs.end()) { + // There's already a pending insert job for this label, add another one (without + // overwrite, only possible in multi index) + assert(this->backendIndex->isMultiValue()); + it->second.push_back(job); + } else { + vecsim_stl::vector new_jobs_vec(1, job, this->allocator); + this->labelToInsertJobs.insert({label, new_jobs_vec}); + } + return job; + } + // Buffer a vector in the frontend index as training data for the backend (re)initialization. // Returns 1 for a new label, 0 for an overwrite, or nullopt if the backend turned ready while // the guard was taken @@ -1242,6 +1246,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::shared_lock lock(this->updateJobMutex); bool flat_holds_old = false; + bool had_pending = false; std::vector taken_over; { std::lock_guard flat_lock{this->flatIndexGuard}; @@ -1255,15 +1260,14 @@ class TieredSVSIndex : public VecSimTieredIndex { return VecSimRelabel_NewLabelTaken; } + // invalid job and rebuild once the label is settled. auto pending = this->labelToInsertJobs.find(old_label); if (pending != this->labelToInsertJobs.end()) { - auto jobs = std::move(pending->second); - this->labelToInsertJobs.erase(pending); - for (auto *job : jobs) { - waitForJob(static_cast(job)); - job->label = new_label; + for (auto *job : pending->second) { + job->id = this->setAndSaveInvalidJob(job); } - this->labelToInsertJobs.emplace(new_label, std::move(jobs)); + this->labelToInsertJobs.erase(pending); + had_pending = true; } // isLabelExists() said new_label is free, but SVS keeps the translator entry of a @@ -1285,27 +1289,29 @@ class TieredSVSIndex : public VecSimTieredIndex { svs_index->consolidate(taken_over); } - const VecSimRelabelCode backend_ret = - this->backendIndex->relabelVector(old_label, new_label); - if (backend_ret == VecSimRelabel_NewLabelTaken) { - // Only reachable if new_label was added concurrently with this rename + const bool rollback = this->backendIndex->relabelVector(old_label, new_label) == + VecSimRelabel_NewLabelTaken; + const labelType final_label = rollback ? old_label : new_label; + + vecsim_stl::vector new_jobs(this->allocator); + if (rollback || had_pending) { std::lock_guard flat_lock{this->flatIndexGuard}; - auto pending = this->labelToInsertJobs.find(new_label); - if (pending != this->labelToInsertJobs.end()) { - auto jobs = std::move(pending->second); - this->labelToInsertJobs.erase(pending); - for (auto *job : jobs) { - waitForJob(static_cast(job)); - job->label = old_label; - } - this->labelToInsertJobs.emplace(old_label, std::move(jobs)); - } - if (flat_holds_old) { + if (rollback && flat_holds_old) { this->frontendIndex->relabelVector(new_label, old_label); } - return VecSimRelabel_NewLabelTaken; + // Rebuild the jobs invalidated above from the buffer itself + if (had_pending) { + for (idType id : this->frontendIndex->getElementIds(final_label)) { + if (ids_to_init_.count(id) == 0) { + new_jobs.push_back(createInsertJob(final_label, id)); + } + } + } + } + for (auto *job : new_jobs) { + this->submitSingleJob(job); } - return VecSimRelabel_OK; + return rollback ? VecSimRelabel_NewLabelTaken : VecSimRelabel_OK; } #endif // HAVE_SVS_REPLACE_EXTERNAL_ID From f5ab8da62a3a8581baa1307c9b7606959bca97e7 Mon Sep 17 00:00:00 2001 From: ethanglaser <42726565+ethanglaser@users.noreply.github.com> Date: Wed, 16 Sep 2026 09:14:44 -0700 Subject: [PATCH 62/64] bump glibc version error message to 2.28 --- cmake/svs.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/svs.cmake b/cmake/svs.cmake index b9aef3431..79baa152e 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -81,7 +81,7 @@ if(USE_SVS) set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc2-reduced.tar.gz" CACHE STRING "SVS URL") endif() else() - message(STATUS "GLIBC >= 2.26 is required for GCC SVS binaries - disabling SVS_SHARED_LIB") + message(STATUS "GLIBC >= 2.28 is required for GCC SVS binaries - disabling SVS_SHARED_LIB") set(SVS_SHARED_LIB OFF) endif() endif() From b5321110785ef69d09097870c497b70782888eed Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 09:19:16 -0700 Subject: [PATCH 63/64] format --- src/VecSim/algorithms/svs/svs_tiered.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 376cda726..8509b6e91 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -798,8 +798,7 @@ class TieredSVSIndex : public VecSimTieredIndex { // if a concurrent deletion drops the instance // initializing it from this single point would refit the compression - const int added = - svs_index->addVectorsIfInitialized(blob_copy.get(), &job->label, 1); + const int added = svs_index->addVectorsIfInitialized(blob_copy.get(), &job->label, 1); job->executing.store(false, std::memory_order_release); if (added == SVSIndexBase::kNotInitialized) { @@ -1289,8 +1288,8 @@ class TieredSVSIndex : public VecSimTieredIndex { svs_index->consolidate(taken_over); } - const bool rollback = this->backendIndex->relabelVector(old_label, new_label) == - VecSimRelabel_NewLabelTaken; + const bool rollback = + this->backendIndex->relabelVector(old_label, new_label) == VecSimRelabel_NewLabelTaken; const labelType final_label = rollback ? old_label : new_label; vecsim_stl::vector new_jobs(this->allocator); From ca35f7c72b46f382fbb86fa66c46733f59f07a4c Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Wed, 16 Sep 2026 10:24:53 -0700 Subject: [PATCH 64/64] fix Relabel duplicates in-flight ingested vectors --- src/VecSim/algorithms/svs/svs_tiered.h | 88 +++++++++++++++++++------- 1 file changed, 64 insertions(+), 24 deletions(-) diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 8509b6e91..3d7c1fa75 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -19,7 +19,11 @@ * Definition of a job that inserts a new vector from flat into SVS Index. */ struct SVSInsertJob : public TieredInsertJob { - std::atomic executing{false}; + // Pending -> Executing -> Done. Delayed means a relabel is remapping the label in the backend + // right now, and publishing would put the vector under a label that is not settled yet. + // Written under flatIndexGuard, except Executing/Done which waitForJob() reads unlocked. + enum class Status : uint8_t { Pending, Delayed, Executing, Done }; + std::atomic status{Status::Pending}; SVSInsertJob(std::shared_ptr allocator, labelType label_, idType id_, JobCallback insertCb, VecSimIndex *index_) @@ -748,8 +752,13 @@ class TieredSVSIndex : public VecSimTieredIndex { // Wait until the job leaves its publish window // Safe to call while holding flatIndexGuard exclusive - template - static void waitForJob(Job *job) { + static void waitForJob(SVSInsertJob *job) { + while (job->status.load(std::memory_order_acquire) == SVSInsertJob::Status::Executing) { + std::this_thread::yield(); + } + } + + static void waitForJob(SVSConsolidateJob *job) { while (job->executing.load(std::memory_order_acquire)) { std::this_thread::yield(); } @@ -778,6 +787,12 @@ class TieredSVSIndex : public VecSimTieredIndex { return InsertJobOutcome::Completed; } + if (job->status.load(std::memory_order_relaxed) == SVSInsertJob::Status::Delayed) { + this->flatIndexGuard.unlock_shared(); + // relabelVector() clears it once the backend remap settled the label + return InsertJobOutcome::Deferred; + } + // a job never inits the backend: one point cannot train the compression if (!svs_index->ready()) { this->flatIndexGuard.unlock_shared(); @@ -793,15 +808,17 @@ class TieredSVSIndex : public VecSimTieredIndex { auto blob_copy = this->getAllocator()->allocate_unique(data_size); memcpy(blob_copy.get(), this->frontendIndex->getDataByInternalId(job->id), data_size); - job->executing.store(true, std::memory_order_release); + job->status.store(SVSInsertJob::Status::Executing, std::memory_order_release); this->flatIndexGuard.unlock_shared(); // if a concurrent deletion drops the instance // initializing it from this single point would refit the compression const int added = svs_index->addVectorsIfInitialized(blob_copy.get(), &job->label, 1); + const bool published = added != SVSIndexBase::kNotInitialized; - job->executing.store(false, std::memory_order_release); - if (added == SVSIndexBase::kNotInitialized) { + job->status.store(published ? SVSInsertJob::Status::Done : SVSInsertJob::Status::Pending, + std::memory_order_release); + if (!published) { return adoptJobIntoInitBuffer(job); } // Remove the vector and the insert job from the flat buffer. @@ -1245,7 +1262,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::shared_lock lock(this->updateJobMutex); bool flat_holds_old = false; - bool had_pending = false; + bool delayed_any = false; std::vector taken_over; { std::lock_guard flat_lock{this->flatIndexGuard}; @@ -1259,14 +1276,25 @@ class TieredSVSIndex : public VecSimTieredIndex { return VecSimRelabel_NewLabelTaken; } - // invalid job and rebuild once the label is settled. auto pending = this->labelToInsertJobs.find(old_label); if (pending != this->labelToInsertJobs.end()) { - for (auto *job : pending->second) { - job->id = this->setAndSaveInvalidJob(job); - } + auto jobs = std::move(pending->second); this->labelToInsertJobs.erase(pending); - had_pending = true; + for (auto *job : jobs) { + auto *insert_job = static_cast(job); + // wait it out: a job that published before the remap is remapped with the + // backend, and still drops its own buffer copy under the new label + waitForJob(insert_job); + job->label = new_label; + if (insert_job->status.load(std::memory_order_relaxed) == + SVSInsertJob::Status::Pending) { + // did not publish yet - hold it until the remap settles the label + insert_job->status.store(SVSInsertJob::Status::Delayed, + std::memory_order_relaxed); + delayed_any = true; + } + } + this->labelToInsertJobs.emplace(new_label, std::move(jobs)); } // isLabelExists() said new_label is free, but SVS keeps the translator entry of a @@ -1292,24 +1320,36 @@ class TieredSVSIndex : public VecSimTieredIndex { this->backendIndex->relabelVector(old_label, new_label) == VecSimRelabel_NewLabelTaken; const labelType final_label = rollback ? old_label : new_label; - vecsim_stl::vector new_jobs(this->allocator); - if (rollback || had_pending) { + if (rollback || delayed_any) { std::lock_guard flat_lock{this->flatIndexGuard}; - if (rollback && flat_holds_old) { - this->frontendIndex->relabelVector(new_label, old_label); + if (rollback) { + auto pending = this->labelToInsertJobs.find(new_label); + if (pending != this->labelToInsertJobs.end()) { + auto jobs = std::move(pending->second); + this->labelToInsertJobs.erase(pending); + for (auto *job : jobs) { + job->label = old_label; + } + this->labelToInsertJobs.emplace(old_label, std::move(jobs)); + } + if (flat_holds_old) { + this->frontendIndex->relabelVector(new_label, old_label); + } } - // Rebuild the jobs invalidated above from the buffer itself - if (had_pending) { - for (idType id : this->frontendIndex->getElementIds(final_label)) { - if (ids_to_init_.count(id) == 0) { - new_jobs.push_back(createInsertJob(final_label, id)); + if (delayed_any) { + auto it = this->labelToInsertJobs.find(final_label); + if (it != this->labelToInsertJobs.end()) { + for (auto *job : it->second) { + auto *insert_job = static_cast(job); + if (insert_job->status.load(std::memory_order_relaxed) == + SVSInsertJob::Status::Delayed) { + insert_job->status.store(SVSInsertJob::Status::Pending, + std::memory_order_relaxed); + } } } } } - for (auto *job : new_jobs) { - this->submitSingleJob(job); - } return rollback ? VecSimRelabel_NewLabelTaken : VecSimRelabel_OK; }