diff --git a/cmake/svs.cmake b/cmake/svs.cmake index fa28df32c..79baa152e 100644 --- a/cmake/svs.cmake +++ b/cmake/svs.cmake @@ -64,7 +64,7 @@ if(USE_SVS) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.31" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "21.0") if(GLIBCXX_VERSION VERSION_GREATER_EQUAL "11") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.3.2/svs-shared-library-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc2-reduced-clang21-gcc11.tar.gz" CACHE STRING "SVS URL") else() message(STATUS "libstdc++ >= GCC 11 is required for Clang SVS binaries - disabling SVS_SHARED_LIB") set(SVS_SHARED_LIB OFF) @@ -76,14 +76,12 @@ if(USE_SVS) else() if(GLIBC_VERSION VERSION_GREATER_EQUAL "2.28") if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "14.0") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.3.2/svs-shared-library-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc2-reduced-gcc14.tar.gz" CACHE STRING "SVS URL") else() - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.3.2/svs-shared-library-reduced.tar.gz" CACHE STRING "SVS URL") + set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/nightly/svs-shared-library-0.5.0rc2-reduced.tar.gz" CACHE STRING "SVS URL") endif() - elseif(GLIBC_VERSION VERSION_GREATER_EQUAL "2.26") - set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.3.0/svs-shared-library-0.3.0-reduced-glibc2_26.tar.gz" CACHE STRING "SVS URL") else() - message(STATUS "GLIBC >= 2.26 is required for GCC SVS binaries - disabling SVS_SHARED_LIB") + message(STATUS "GLIBC >= 2.28 is required for GCC SVS binaries - disabling SVS_SHARED_LIB") set(SVS_SHARED_LIB OFF) endif() endif() @@ -128,7 +126,7 @@ if(USE_SVS) # so whether it is available depends on which SVS this build ended up with rather than on the # platform. Detect it in the header instead of assuming: it is a method, not a file, so this # greps the header the LVQ check would have tested for existence. - set(SVS_DYNAMIC_INDEX_HEADER "svs/index/vamana/dynamic_index.h") + set(SVS_DYNAMIC_INDEX_HEADER "svs/concurrent/dynamic_index.h") set(SVS_HAS_REPLACE_EXTERNAL_ID 0) if(EXISTS "${svs_SOURCE_DIR}/include/${SVS_DYNAMIC_INDEX_HEADER}") file(READ "${svs_SOURCE_DIR}/include/${SVS_DYNAMIC_INDEX_HEADER}" SVS_DYNAMIC_INDEX_SRC) diff --git a/deps/ScalableVectorSearch b/deps/ScalableVectorSearch index a7e3494fa..3ee890d6d 160000 --- a/deps/ScalableVectorSearch +++ b/deps/ScalableVectorSearch @@ -1 +1 @@ -Subproject commit a7e3494faab9454f577b87ac934a96c78c45dd7b +Subproject commit 3ee890d6d52102e979ebd4331823b6236ec17a56 diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index 2ce673343..9180b91ff 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -24,13 +24,10 @@ /** * Definition of a job that inserts a new vector from flat into HNSW Index. */ -struct HNSWInsertJob : public AsyncJob { - labelType label; - idType id; - +struct HNSWInsertJob : public TieredInsertJob { HNSWInsertJob(std::shared_ptr allocator, labelType label_, idType id_, JobCallback insertCb, VecSimIndex *index_) - : AsyncJob(allocator, HNSW_INSERT_VECTOR_JOB, insertCb, index_), label(label_), id(id_) {} + : TieredInsertJob(allocator, HNSW_INSERT_VECTOR_JOB, label_, id_, insertCb, index_) {} }; /** @@ -81,17 +78,10 @@ template class TieredHNSWIndex : public VecSimTieredIndex { private: /// Mappings from id/label to associated jobs, for invalidating and update ids if necessary. - // In MULTI, we can have more than one insert job pending per label. // **This map is protected with the flat buffer lock** - vecsim_stl::unordered_map> labelToInsertJobs; vecsim_stl::unordered_map> idToRepairJobs; vecsim_stl::unordered_map idToSwapJob; - // A mapping to hold invalid jobs, so we can dispose them upon index deletion. - vecsim_stl::unordered_map invalidJobs; - idType currInvalidJobId; // A unique arbitrary identifier for accessing invalid jobs - std::mutex invalidJobsLookupGuard; - // This threshold is tested upon deleting a label from HNSW, and once the number of deleted // vectors reached this limit, we apply swap jobs *only for vectors that has no more pending // repair jobs*, and are ready to be removed from the graph. @@ -172,13 +162,6 @@ class TieredHNSWIndex : public VecSimTieredIndex { inline HNSWIndex *getHNSWIndex() const; - // Helper function for deleting a vector from the flat buffer (after it has already been - // ingested into HNSW or deleted). This includes removing the corresponding insert job from the - // label-to-insert-jobs lookup. Also, since deletion a vector triggers swapping of the - // internal last id with the deleted vector id, here we update the pending insert job(s) for the - // last id (if needed). This should be called while *flat lock is held* (exclusive lock). - void updateInsertJobInternalId(idType prev_id, idType new_id, labelType label); - // Helper function for performing in place mark delete of vector(s) associated with a label // and creating the appropriate repair jobs for the effected connections. This should be called // while *HNSW shared lock is held* (shared locked). @@ -197,11 +180,6 @@ class TieredHNSWIndex : public VecSimTieredIndex { void insertVectorToHNSW(HNSWIndex *hnsw_index, labelType label, const void *blob); - // Set an insert/repair job as invalid, put the job pointer in the invalid jobs lookup under - // the current available id, increase it and return it (while holding invalidJobsLookupGuard). - // Returns the id that the job was stored under (to be set in the job id field). - idType setAndSaveInvalidJob(AsyncJob *job); - // Handle deletion of vector inplace considering that async deletion might occurred beforehand. int deleteLabelFromHNSWInplace(labelType label); @@ -209,6 +187,31 @@ class TieredHNSWIndex : public VecSimTieredIndex { #include "VecSim/algorithms/hnsw/hnsw_tiered_tests_friends.h" #endif +protected: + // Adapts HNSW's own data-guard mutex to Lockable + class HnswDataGuardLockable : public Lockable { + HNSWIndex *hnsw; + + public: + explicit HnswDataGuardLockable(HNSWIndex *hnsw) : hnsw(hnsw) {} + void lock() const override { hnsw->lockSharedIndexDataGuard(); } + void unlock() const override { hnsw->unlockSharedIndexDataGuard(); } + }; + HnswDataGuardLockable hnswDataGuardLockable{this->getHNSWIndex()}; + + ScopedLocks lockMainIndexForQuery() const override { + return ScopedLocks(this->mainIndexLockable); + } + + ScopedLocks lockIndexForSize() const override { + return ScopedLocks(this->flatIndexLockable, hnswDataGuardLockable); + } + + ScopedLocks lockIndexForCapacity() const override { + // No-op: No locking required for capacity() + return ScopedLocks(); + } + public: class TieredHNSW_BatchIterator : public VecSimBatchIterator { private: @@ -285,9 +288,6 @@ class TieredHNSWIndex : public VecSimTieredIndex { int deleteVector(labelType label) override; VecSimRelabelCode relabelVector(labelType old_label, labelType new_label) override; size_t getNumMarkedDeleted() const override { return getHNSWIndex()->getNumMarkedDeleted(); } - size_t indexSize() const override; - size_t indexCapacity() const override; - double getDistanceFrom_Unsafe(labelType label, const void *blob) const override; // Do nothing here, each tier (flat buffer and HNSW) should increase capacity for itself when // needed. VecSimIndexDebugInfo debugInfo() const override; @@ -341,16 +341,16 @@ class TieredHNSWIndex : public VecSimTieredIndex { /* Helper methods */ template void TieredHNSWIndex::executeInsertJobWrapper(AsyncJob *job) { - auto *insert_job = reinterpret_cast(job); - auto *job_index = reinterpret_cast *>(insert_job->index); + auto *insert_job = static_cast(job); + auto *job_index = static_cast *>(insert_job->index); job_index->executeInsertJob(insert_job); delete job; } template void TieredHNSWIndex::executeRepairJobWrapper(AsyncJob *job) { - auto *repair_job = reinterpret_cast(job); - auto *job_index = reinterpret_cast *>(repair_job->index); + auto *repair_job = static_cast(job); + auto *job_index = static_cast *>(repair_job->index); job_index->executeRepairJob(repair_job); delete job; } @@ -517,22 +517,6 @@ int TieredHNSWIndex::deleteLabelFromHNSW(labelType label) { return internal_ids.size(); } -template -void TieredHNSWIndex::updateInsertJobInternalId(idType prev_id, idType new_id, - labelType label) { - // Update the pending job id, due to a swap that was caused after the removal of new_id. - assert(new_id != INVALID_ID && prev_id != INVALID_ID); - auto it = this->labelToInsertJobs.find(label); - if (it != this->labelToInsertJobs.end()) { - // There is a pending job for the label of the swapped last id - update its id. - for (HNSWInsertJob *job_it : it->second) { - if (job_it->id == prev_id) { - job_it->id = new_id; - } - } - } -} - template template void TieredHNSWIndex::insertVectorToHNSW( @@ -602,16 +586,6 @@ void TieredHNSWIndex::insertVectorToHNSW( } } -template -idType TieredHNSWIndex::setAndSaveInvalidJob(AsyncJob *job) { - this->invalidJobsLookupGuard.lock(); - job->isValid = false; - idType curInvalidId = currInvalidJobId++; - this->invalidJobs.insert({curInvalidId, job}); - this->invalidJobsLookupGuard.unlock(); - return curInvalidId; -} - template int TieredHNSWIndex::deleteLabelFromHNSWInplace(labelType label) { auto *hnsw_index = this->getHNSWIndex(); @@ -669,40 +643,7 @@ void TieredHNSWIndex::executeInsertJob(HNSWInsertJob *job) { #endif // Remove the vector and the insert job from the flat buffer. - this->flatIndexGuard.lock(); - // The job might have been invalidated due to overwrite in the meantime. In this case, - // it was already deleted and the job has been evicted. Otherwise, we need to do it now. - if (job->isValid) { - // Remove the job pointer from the labelToInsertJobs mapping. - auto &jobs = labelToInsertJobs.at(job->label); - for (size_t i = 0; i < jobs.size(); i++) { - if (jobs[i]->id == job->id) { - jobs.erase(jobs.begin() + (long)i); - break; - } - } - if (labelToInsertJobs.at(job->label).empty()) { - labelToInsertJobs.erase(job->label); - } - // Remove the vector from the flat buffer. This may cause the last vector id to swap with - // the deleted id. Hold the label for the last id, so we can later on update its - // corresponding job id. Note that after calling deleteVectorById, the last id's label - // shouldn't be available, since it is removed from the lookup. - labelType last_vec_label = - this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); - int deleted = this->frontendIndex->deleteVectorById(job->label, job->id); - if (deleted && job->id != this->frontendIndex->indexSize()) { - // If the vector removal caused a swap with the last id, update the relevant insert job. - this->updateInsertJobInternalId(this->frontendIndex->indexSize(), job->id, - last_vec_label); - } - } else { - // Remove the current job from the invalid jobs' lookup, as we are about to delete it now. - this->invalidJobsLookupGuard.lock(); - this->invalidJobs.erase(job->id); - this->invalidJobsLookupGuard.unlock(); - } - this->flatIndexGuard.unlock(); + this->removeIngestedVectorFromFlat(job); } template @@ -773,9 +714,7 @@ TieredHNSWIndex::TieredHNSWIndex(HNSWIndex allocator) : VecSimTieredIndex(hnsw_index, bf_index, tiered_index_params, allocator), - labelToInsertJobs(this->allocator), idToRepairJobs(this->allocator), - idToSwapJob(this->allocator), invalidJobs(this->allocator), currInvalidJobId(0), - readySwapJobs(0), + idToRepairJobs(this->allocator), idToSwapJob(this->allocator), readySwapJobs(0), isQuantized(tiered_index_params.primaryIndexParams->algoParams.hnswParams.quantType != VecSimQuant_NONE) { const auto &hnsw_params = tiered_index_params.primaryIndexParams->algoParams.hnswParams; @@ -795,12 +734,6 @@ TieredHNSWIndex::TieredHNSWIndex(HNSWIndex TieredHNSWIndex::~TieredHNSWIndex() { - // Delete all the pending insert jobs. - for (auto &jobs : this->labelToInsertJobs) { - for (auto *job : jobs.second) { - delete job; - } - } // Delete all the pending repair jobs. for (auto &jobs : this->idToRepairJobs) { for (auto *job : jobs.second) { @@ -811,10 +744,6 @@ TieredHNSWIndex::~TieredHNSWIndex() { for (auto &it : this->idToSwapJob) { delete it.second; } - // Delete all the pending invalid jobs. - for (auto &it : this->invalidJobs) { - delete it.second; - } } // Training writes stay in FLAT in both write modes, even beyond flatBufferLimit. @@ -825,20 +754,21 @@ int TieredHNSWIndex::addVectorDuringAccumulation(const void const auto [result, count, should_finalize] = [&] { std::lock_guard flat_lock(this->flatIndexGuard); idType id = this->frontendIndex->indexSize(); - HNSWInsertJob *job = nullptr; + TieredInsertJob *job = nullptr; if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { id = static_cast *>(this->frontendIndex) ->getIdOfLabel(label); subtractFromSum( {this->frontendIndex->getDataByInternalId(id), this->frontendIndex->getDim()}); - job = labelToInsertJobs.at(label).front(); + job = this->labelToInsertJobs.at(label).front(); } const int result = this->frontendIndex->addVector(blob, label); addToSum({this->frontendIndex->getDataByInternalId(id), this->frontendIndex->getDim()}); if (!job) { job = new (this->allocator) HNSWInsertJob(this->allocator, label, id, executeInsertJobWrapper, this); - auto [it, inserted] = labelToInsertJobs.try_emplace(label, 1, job, this->allocator); + auto [it, inserted] = + this->labelToInsertJobs.try_emplace(label, 1, job, this->allocator); if (!inserted) { it->second.push_back(job); } @@ -857,8 +787,8 @@ int TieredHNSWIndex::addVectorDuringAccumulation(const void template int TieredHNSWIndex::deleteVectorDuringAccumulation(labelType label) { std::lock_guard flat_lock(this->flatIndexGuard); - auto it = labelToInsertJobs.find(label); - if (it == labelToInsertJobs.end()) { + auto it = this->labelToInsertJobs.find(label); + if (it == this->labelToInsertJobs.end()) { return 0; } const int removed = it->second.size(); @@ -867,10 +797,10 @@ int TieredHNSWIndex::deleteVectorDuringAccumulation(labelTyp {this->frontendIndex->getDataByInternalId(job->id), this->frontendIndex->getDim()}); delete job; // Never submitted, so no worker can still reference this job. } - labelToInsertJobs.erase(it); + this->labelToInsertJobs.erase(it); auto updated_ids = this->frontendIndex->deleteVectorAndGetUpdatedIds(label); for (const auto &[new_id, previous] : updated_ids) { - updateInsertJobInternalId(previous.first, new_id, previous.second); + this->updateInsertJobInternalId(previous.first, new_id, previous.second); } return removed; } @@ -897,7 +827,7 @@ void TieredHNSWIndex::finalizeQuantizationAndSubmitJobs(size jobs.reserve(count); { std::shared_lock flat_lock(this->flatIndexGuard); - for (const auto &[label, pending] : labelToInsertJobs) { + for (const auto &[label, pending] : this->labelToInsertJobs) { jobs.insert(jobs.end(), pending.begin(), pending.end()); } } @@ -926,20 +856,6 @@ void TieredHNSWIndex::finalizeQuantizationAndSubmitJobs(size } } -template -size_t TieredHNSWIndex::indexSize() const { - std::shared_lock flat_index_lock(this->flatIndexGuard); - const size_t res = this->frontendIndex->indexSize(); - auto *hnsw_index = getHNSWIndex(); - auto index_data_lock = hnsw_index->acquireSharedIndexDataGuard(); - return res + hnsw_index->indexSize(); -} - -template -size_t TieredHNSWIndex::indexCapacity() const { - return this->backendIndex->indexCapacity() + this->frontendIndex->indexCapacity(); -} - // In the tiered index, we assume that the blobs are processed by the flat buffer // before being transferred to the HNSW index. // When inserting vectors directly into the HNSW index—such as in VecSim_WriteInPlace mode— or when @@ -1168,7 +1084,7 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o if (jobs_it != this->labelToInsertJobs.end()) { auto jobs = std::move(jobs_it->second); this->labelToInsertJobs.erase(jobs_it); - for (HNSWInsertJob *job : jobs) { + for (TieredInsertJob *job : jobs) { job->label = new_label; } this->labelToInsertJobs.emplace(new_label, std::move(jobs)); @@ -1199,50 +1115,6 @@ VecSimRelabelCode TieredHNSWIndex::relabelVector(labelType o return VecSimRelabel_OK; } -// `getDistanceFrom` returns the minimum distance between the given blob and the vector with the -// given label. If the label doesn't exist, the distance will be NaN. -// Therefore, it's better to just call `getDistanceFrom` on both indexes and return the minimum -// instead of checking if the label exists in each index. We first try to get the distance from the -// flat buffer, as vectors in the buffer might move to the Main while we're "between" the locks. -// Behavior for single (regular) index: -// 1. label doesn't exist in both indexes - return NaN -// 2. label exists in one of the indexes only - return the distance from that index (which is valid) -// 3. label exists in both indexes - return the value from the flat buffer (which is valid and equal -// to the value from the Main index), saving us from locking the Main index. -// Behavior for multi index: -// 1. label doesn't exist in both indexes - return NaN -// 2. label exists in one of the indexes only - return the distance from that index (which is valid) -// 3. label exists in both indexes - we may have some of the vectors with the same label in the flat -// buffer only and some in the Main index only (and maybe temporal duplications). -// So, we get the distance from both indexes and return the minimum. - -// IMPORTANT: this should be called when the *tiered index locks are locked for shared ownership*, -// along with HNSW index data guard lock. That is since the internal getDistanceFrom calls access -// the indexes' data, and it is not safe to run insert/delete operation in parallel. Also, we avoid -// acquiring the locks internally, since this is usually called for every vector individually, and -// the overhead of acquiring and releasing the locks is significant in that case. -template -double TieredHNSWIndex::getDistanceFrom_Unsafe(labelType label, - const void *blob) const { - // Try to get the distance from the flat buffer. - // If the label doesn't exist, the distance will be NaN. - auto flat_dist = this->frontendIndex->getDistanceFrom_Unsafe(label, blob); - - // Optimization. TODO: consider having different implementations for single and multi indexes, - // to avoid checking the index type on every query. - if (!this->backendIndex->isMultiValue() && !std::isnan(flat_dist)) { - // If the index is single value, and we got a valid distance from the flat buffer, - // we can return the distance without querying the Main index. - return flat_dist; - } - - // Try to get the distance from the Main index. - auto hnsw_dist = getHNSWIndex()->getDistanceFrom_Unsafe(label, blob); - - // Return the minimum distance that is not NaN. - return std::fmin(flat_dist, hnsw_dist); -} - //////////////////////////////////////////////////////////////////////////////////////////////////// // TieredHNSW_BatchIterator // //////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index d6c4566cb..dfddb7dc0 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include #include @@ -38,16 +40,28 @@ struct SVSIndexBase { SVSIndexBase() : num_marked_deleted{0} {}; virtual ~SVSIndexBase() = default; + // Returned by addVectorsIfInitialized() when nothing was added. + static constexpr int kNotInitialized = -1; + virtual int addVector(const void *vector_data, labelType label) = 0; virtual int addVectors(const void *vectors_data, const labelType *labels, size_t n) = 0; + // Add vectors into an already initialized instance. + virtual int addVectorsIfInitialized(const void *vectors_data, const labelType *labels, + size_t n) = 0; + virtual int deleteVector(labelType label) = 0; virtual int deleteVectors(const labelType *labels, size_t n) = 0; + virtual void consolidate(const std::vector &labels) = 0; virtual bool isLabelExists(labelType label) const = 0; virtual size_t indexStorageSize() const = 0; virtual size_t getParallelism() const = 0; virtual void setParallelism(size_t parallelism) = 0; virtual size_t getPoolSize() const = 0; virtual bool isCompressed() const = 0; - size_t getNumMarkedDeleted() const { return num_marked_deleted; } + virtual bool ready() const = 0; + + size_t getNumMarkedDeleted() const { + return num_marked_deleted.load(std::memory_order_relaxed); + } // Abstract handler to manage SVS implementation instance // declared to avoid unsafe unique_ptr usage @@ -64,7 +78,7 @@ struct SVSIndexBase protected: // Index marked deleted vectors counter to initiate reindexing if it exceeds threshold // markIndexUpdate() manages this counter - size_t num_marked_deleted; + std::atomic num_marked_deleted; }; /** Thread Management Strategy: @@ -88,10 +102,12 @@ class SVSIndex : public VecSimIndexAbstract, fl using graph_builder_t = SVSGraphBuilder; using graph_type = typename graph_builder_t::graph_type; + // Note. svs::concurrent::* classes differ from the svs::index::vamana::* classes + // in their mutation contracts. using impl_type = std::conditional_t< isMulti, - svs::index::vamana::MultiMutableVamanaIndex, - svs::index::vamana::MutableVamanaIndex>; + svs::concurrent::MultiMutableVamanaIndex, + svs::concurrent::MutableVamanaIndex>; bool forcePreprocessing; @@ -113,8 +129,21 @@ class SVSIndex : public VecSimIndexAbstract, fl // SVS thread pool VecSimSVSThreadPool threadpool_; svs::logging::logger_ptr logger_; - // SVS Index implementation instance - std::unique_ptr impl_; + // SVS Index implementation instance. + // initImpl() needs at least one point to compute an entry point. + // so impl_ is null until the first insert + std::shared_ptr impl_; + + mutable std::shared_mutex pimplGuard_; + // Serialize mutators of the impl_ pointer (exclusive) + // vs add_points() (shared) to avoid silent drops of additions + // Lock order: implMutationGuard_ -> pimplGuard_. + std::shared_mutex implMutationGuard_; + + std::shared_ptr getImpl() const { + std::shared_lock lock(this->pimplGuard_); + return this->impl_; + } static double toVecSimDistance(float v) { return svs_details::toVecSimDistance(v); } @@ -157,7 +186,7 @@ class SVSIndex : public VecSimIndexAbstract, fl // Create SVS index instance with initial data // Data should not be empty template - std::unique_ptr initImpl(const Dataset &points, + std::shared_ptr initImpl(const Dataset &points, std::span ids) const { svs::threads::ThreadPoolHandle threadpool_handle{VecSimSVSThreadPool{threadpool_}}; @@ -178,7 +207,7 @@ class SVSIndex : public VecSimIndexAbstract, fl this->blockSize, this->getAllocator(), logger_); // Create SVS MutableIndex instance - auto impl = std::make_unique(std::move(graph), std::move(data), entry_point, + auto impl = std::make_shared(std::move(graph), std::move(data), entry_point, std::move(distance), ids, threadpool_, logger_); // Set SVS MutableIndex build parameters to be used in future updates @@ -221,8 +250,8 @@ class SVSIndex : public VecSimIndexAbstract, fl // Handler to manage SVS implementation instance struct SVSImplHandler : public SVSIndexBase::ImplHandler { - std::unique_ptr impl; - SVSImplHandler(std::unique_ptr impl) : impl{std::move(impl)} {} + std::shared_ptr impl; + SVSImplHandler(std::shared_ptr impl) : impl{std::move(impl)} {} }; std::unique_ptr createImpl(const void *vectors_data, const labelType *labels, @@ -242,14 +271,17 @@ class SVSIndex : public VecSimIndexAbstract, fl } void setImpl(std::unique_ptr handler) override { - if (impl_ != nullptr) { - throw std::logic_error("SVSIndex::setImpl called on non-empty impl_"); - } - SVSImplHandler *svs_handler = dynamic_cast(handler.get()); if (!svs_handler) { throw std::logic_error("Failed to cast to SVSImplHandler"); } + + std::lock_guard replace_lock(this->implMutationGuard_); + if (getImpl()) { + throw std::logic_error("SVSIndex::setImpl called on non-empty impl_"); + } + + std::lock_guard lock(this->pimplGuard_); this->impl_ = std::move(svs_handler->impl); } @@ -259,17 +291,13 @@ class SVSIndex : public VecSimIndexAbstract, fl // for the operation. // Important NOTE: For single vector operations (n=1), parallelism should be 1. // For bulk operations (n>1), parallelism should reflect the number of available threads. - int addVectorsImpl(const void *vectors_data, const labelType *labels, size_t n) { + int addVectorsImpl(const void *vectors_data, const labelType *labels, size_t n, + bool allow_init = true) { if (n == 0) { return 0; } int deleted_num = 0; - if constexpr (!isMulti) { - // SVS index does not support overriding vectors with the same label - // so we have to delete them first if needed - deleted_num = deleteVectorsImpl(labels, n); - } std::span ids(labels, n); auto processed_blob = this->preprocessForBatchStorage(vectors_data, n); @@ -277,65 +305,93 @@ class SVSIndex : public VecSimIndexAbstract, fl // Wrap data into SVS SimpleDataView for SVS API auto points = svs::data::SimpleDataView{typed_vectors_data, n, this->dim}; - if (!impl_) { - // SVS index instance cannot be empty, so we have to construct it at first rows - impl_ = initImpl(points, ids); + if constexpr (!isMulti) { + // SVS index does not support overriding vectors with the same label + // so we have to delete them first if needed + deleted_num = deleteVectorsImpl(labels, n); + } + + this->implMutationGuard_.lock_shared(); + if (auto impl = getImpl()) { + impl->add_points(points, ids, /*reuse_empty*/ false); + this->implMutationGuard_.unlock_shared(); } else { - // Add new points to existing SVS index - impl_->add_points(points, ids); + this->implMutationGuard_.unlock_shared(); + this->implMutationGuard_.lock(); + if (auto existing = getImpl()) { + existing->add_points(points, ids, /*reuse_empty*/ false); + } else if (!allow_init) { + this->implMutationGuard_.unlock(); + return kNotInitialized; + } else { + auto built = initImpl(points, ids); + assert(built != nullptr); + std::lock_guard lock(this->pimplGuard_); + this->impl_ = std::move(built); + } + this->implMutationGuard_.unlock(); } return n - deleted_num; } + void consolidate(const std::vector &labels) override { + if (auto impl = getImpl()) { + impl->consolidate(labels); + } + } + int deleteVectorImpl(const labelType label) { - if (indexLabelCount() == 0 || !impl_->has_id(label)) { + auto impl = getImpl(); + if (!impl) { return 0; } - const auto deleted_num = impl_->delete_entries(std::span{&label, 1}); - - this->markIndexUpdate(deleted_num); + const int deleted_num = impl->delete_entries(std::span{&label, 1}); + this->markIndexUpdate(impl, deleted_num); return deleted_num; } int deleteVectorsImpl(const labelType *labels, size_t n) { - if (indexLabelCount() == 0) { + auto impl = getImpl(); + if (!impl) { return 0; } - // SVS fails if we try to delete non-existing entries - std::vector entries_to_delete; - entries_to_delete.reserve(n); - for (size_t i = 0; i < n; i++) { - if (impl_->has_id(labels[i])) { - entries_to_delete.push_back(labels[i]); - } + const int deleted_num = impl->delete_entries(std::span{labels, n}); + if (deleted_num > 0) { + this->markIndexUpdate(impl, deleted_num); } - - if (entries_to_delete.size() == 0) { - return 0; - } - - const auto deleted_num = impl_->delete_entries(entries_to_delete); - - this->markIndexUpdate(deleted_num); return deleted_num; } - // Count deletions and consolidate index if needed - void markIndexUpdate(size_t n = 1) { - if (!impl_) - return; - + void markIndexUpdate(const std::shared_ptr &impl, size_t n = 1) { // SVS index instance should not be empty - if (indexLabelCount() == 0) { - this->impl_.reset(); - num_marked_deleted = 0; - return; + if (labelCountOf(*impl) == 0) { + std::lock_guard replace_lock(this->implMutationGuard_); + if (getImpl() != impl) { + // The instance was already droped + return; + } + if (labelCountOf(*impl) == 0) { + { + std::lock_guard lock(this->pimplGuard_); + this->impl_.reset(); + } + num_marked_deleted.store(0, std::memory_order_relaxed); + return; + } } - num_marked_deleted += n; + num_marked_deleted.fetch_add(n, std::memory_order_relaxed); + } + + static size_t labelCountOf(const impl_type &impl) { + if constexpr (isMulti) { + return impl.labelcount(); + } else { + return impl.size(); + } } bool isTwoLevelLVQ(const VecSimSvsQuantBits &qbits) { @@ -374,26 +430,32 @@ class SVSIndex : public VecSimIndexAbstract, fl ~SVSIndex() = default; + bool ready() const override { + std::shared_lock lock(this->pimplGuard_); + return this->impl_ != nullptr; + } + size_t indexSize() const override { return indexStorageSize(); } - size_t indexStorageSize() const override { return impl_ ? impl_->view_data().size() : 0; } + size_t indexStorageSize() const override { + auto impl = getImpl(); + return impl ? impl->view_data().size() : 0; + } size_t indexCapacity() const override { - return impl_ ? storage_traits_t::storage_capacity(impl_->view_data()) : 0; + auto impl = getImpl(); + return impl ? storage_traits_t::storage_capacity(impl->view_data()) : 0; } size_t indexLabelCount() const override { - if constexpr (isMulti) { - return impl_ ? impl_->labelcount() : 0; - } else { - return impl_ ? impl_->size() : 0; - } + auto impl = getImpl(); + return impl ? labelCountOf(*impl) : 0; } vecsim_stl::set getLabelsSet() const override { vecsim_stl::set labels(this->allocator); - if (impl_) { - impl_->on_ids([&labels](size_t label) { labels.insert(label); }); + if (auto impl = getImpl()) { + impl->on_ids([&labels](size_t label) { labels.insert(label); }); } return labels; } @@ -543,6 +605,13 @@ class SVSIndex : public VecSimIndexAbstract, fl return addVectorsImpl(vectors_data, labels, n); } + int addVectorsIfInitialized(const void *vectors_data, const labelType *labels, + size_t n) override { + assert(!(n == 1 && getParallelism() > 1) && + "Can't use more than one thread to insert a single vector"); + return addVectorsImpl(vectors_data, labels, n, /*allow_init=*/false); + } + int deleteVector(labelType label) override { return deleteVectorImpl(label); } int deleteVectors(const labelType *labels, size_t n) override { @@ -558,22 +627,31 @@ class SVSIndex : public VecSimIndexAbstract, fl if (old_label == new_label) { return VecSimRelabel_SameLabel; } - // `isLabelExists` also covers the index that never held a vector, where `impl_` has not - // been created yet and so trivially holds nothing. - if (!isLabelExists(old_label)) { + + std::shared_lock mutation_lock(this->implMutationGuard_); + auto impl = getImpl(); + + // A null impl_ is an index that never held a vector, so it holds neither label. + if (!impl) { return VecSimRelabel_OldLabelMissing; } - if (isLabelExists(new_label)) { + + // SVS checks both ends under its own translator lock + switch (impl->replace_external_id(old_label, new_label)) { + case svs::concurrent::ReplaceExternalIdResult::Ok: + return VecSimRelabel_OK; + case svs::concurrent::ReplaceExternalIdResult::NewIdExists: return VecSimRelabel_NewLabelTaken; + case svs::concurrent::ReplaceExternalIdResult::OldIdMissing: + break; } - - impl_->replace_external_id(old_label, new_label); - return VecSimRelabel_OK; + return VecSimRelabel_OldLabelMissing; } #endif // HAVE_SVS_REPLACE_EXTERNAL_ID bool isLabelExists(labelType label) const override { - return impl_ ? impl_->has_id(label) : false; + auto impl = getImpl(); + return impl ? impl->has_id(label) : false; } size_t getParallelism() const override { return threadpool_.getParallelism(); } @@ -588,12 +666,13 @@ class SVSIndex : public VecSimIndexAbstract, fl } double getDistanceFrom_Unsafe(labelType label, const void *vector_data) const override { - if (!impl_ || !impl_->has_id(label)) { + auto impl = getImpl(); + if (!impl || !impl->has_id(label)) { return std::numeric_limits::quiet_NaN(); - }; + } auto query_datum = std::span{static_cast(vector_data), this->dim}; - auto dist = impl_->get_distance(label, query_datum); + auto dist = impl->get_distance(label, query_datum); return toVecSimDistance(dist); } @@ -601,39 +680,45 @@ class SVSIndex : public VecSimIndexAbstract, fl VecSimQueryParams *queryParams) const override { auto rep = new VecSimQueryReply(this->allocator); this->lastMode = STANDARD_KNN; - if (k == 0 || this->indexLabelCount() == 0) { + auto impl = getImpl(); + if (k == 0 || !impl) { return rep; } + { + const auto label_count = labelCountOf(*impl); + if (label_count == 0) { + return rep; + } + // limit result size to index size + k = std::min(k, label_count); - // limit result size to index size - k = std::min(k, this->indexLabelCount()); - - auto processed_query_ptr = this->preprocessQuery(queryBlob); - const void *processed_query = processed_query_ptr.get(); + auto processed_query_ptr = this->preprocessQuery(queryBlob); + const void *processed_query = processed_query_ptr.get(); - auto query = svs::data::ConstSimpleDataView{ - static_cast(processed_query), 1, this->dim}; - auto result = svs::QueryResult{query.size(), k}; - auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams, - is_two_level_lvq); + auto query = svs::data::ConstSimpleDataView{ + static_cast(processed_query), 1, this->dim}; + auto result = svs::QueryResult{query.size(), k}; + auto sp = svs_details::joinSearchParams(impl->get_search_parameters(), queryParams, + is_two_level_lvq); - auto timeoutCtx = queryParams ? queryParams->timeoutCtx : nullptr; - auto cancel = [timeoutCtx]() { return VECSIM_TIMEOUT(timeoutCtx); }; + auto timeoutCtx = queryParams ? queryParams->timeoutCtx : nullptr; + auto cancel = [timeoutCtx]() { return VECSIM_TIMEOUT(timeoutCtx); }; - impl_->search(result.view(), query, sp, cancel); - if (cancel()) { - rep->code = VecSim_QueryReply_TimedOut; - return rep; - } + impl->search(result.view(), query, sp, cancel); + if (cancel()) { + rep->code = VecSim_QueryReply_TimedOut; + return rep; + } - assert(result.n_queries() == 1); + assert(result.n_queries() == 1); - const auto n_neighbors = result.n_neighbors(); - rep->results.reserve(n_neighbors); + const auto n_neighbors = result.n_neighbors(); + rep->results.reserve(n_neighbors); - for (size_t i = 0; i < n_neighbors; i++) { - rep->results.push_back( - VecSimQueryResult{result.index(0, i), toVecSimDistance(result.distance(0, i))}); + for (size_t i = 0; i < n_neighbors; i++) { + rep->results.push_back( + VecSimQueryResult{result.index(0, i), toVecSimDistance(result.distance(0, i))}); + } } // Workaround for VecSim merge_results() that expects results to be sorted // by score, then by id from both indices. @@ -646,61 +731,63 @@ class SVSIndex : public VecSimIndexAbstract, fl VecSimQueryParams *queryParams) const override { auto rep = new VecSimQueryReply(this->allocator); this->lastMode = RANGE_QUERY; - if (radius == 0 || this->indexLabelCount() == 0) { - return rep; - } - - auto timeoutCtx = queryParams ? queryParams->timeoutCtx : nullptr; - auto cancel = [timeoutCtx]() { return VECSIM_TIMEOUT(timeoutCtx); }; - - // Prepare query blob for SVS - auto processed_query_ptr = this->preprocessQuery(queryBlob); - const void *processed_query = processed_query_ptr.get(); - std::span query{static_cast(processed_query), - this->dim}; - - // Base search parameters for the SVS iterator schedule. - auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams, - is_two_level_lvq); - // SVS BatchIterator handles the search in batches - // The batch size is set to the index search window size by default - const size_t batch_size = sp.buffer_config_.get_search_window_size(); - - // Create SVS BatchIterator for range search - // Search result is cached in the iterator and can be accessed by the user - auto svs_it = impl_->make_batch_iterator(query); - svs_it.next(batch_size, cancel); - if (cancel()) { - rep->code = VecSim_QueryReply_TimedOut; + auto impl = getImpl(); + if (radius == 0 || !impl || labelCountOf(*impl) == 0) { return rep; } + { + auto timeoutCtx = queryParams ? queryParams->timeoutCtx : nullptr; + auto cancel = [timeoutCtx]() { return VECSIM_TIMEOUT(timeoutCtx); }; + + // Prepare query blob for SVS + auto processed_query_ptr = this->preprocessQuery(queryBlob); + const void *processed_query = processed_query_ptr.get(); + std::span query{static_cast(processed_query), + this->dim}; + + // Base search parameters for the SVS iterator schedule. + auto sp = svs_details::joinSearchParams(impl->get_search_parameters(), queryParams, + is_two_level_lvq); + // SVS BatchIterator handles the search in batches + // The batch size is set to the index search window size by default + const size_t batch_size = sp.buffer_config_.get_search_window_size(); + + // Create SVS BatchIterator for range search + // Search result is cached in the iterator and can be accessed by the user + auto svs_it = impl->make_batch_iterator(query); + svs_it.next(batch_size, cancel); + if (cancel()) { + rep->code = VecSim_QueryReply_TimedOut; + return rep; + } - // range search using epsilon - const auto epsilon = queryParams && queryParams->svsRuntimeParams.epsilon != 0 - ? queryParams->svsRuntimeParams.epsilon - : this->epsilon; - - const auto range_search_boundaries = radius * (1.0 + std::abs(epsilon)); - bool keep_searching = true; - - // Loop while iterator cache is not empty and search radius + epsilon is not exceeded - while (keep_searching && svs_it.size() > 0) { - // Iterate over the cached search results - for (auto &neighbor : svs_it) { - const auto dist = toVecSimDistance(neighbor.distance()); - if (dist <= radius) { - rep->results.push_back(VecSimQueryResult{neighbor.id(), dist}); - } else if (dist > range_search_boundaries) { - keep_searching = false; + // range search using epsilon + const auto epsilon = queryParams && queryParams->svsRuntimeParams.epsilon != 0 + ? queryParams->svsRuntimeParams.epsilon + : this->epsilon; + + const auto range_search_boundaries = radius * (1.0 + std::abs(epsilon)); + bool keep_searching = true; + + // Loop while iterator cache is not empty and search radius + epsilon is not exceeded + while (keep_searching && svs_it.size() > 0) { + // Iterate over the cached search results + for (auto &neighbor : svs_it) { + const auto dist = toVecSimDistance(neighbor.distance()); + if (dist <= radius) { + rep->results.push_back(VecSimQueryResult{neighbor.id(), dist}); + } else if (dist > range_search_boundaries) { + keep_searching = false; + } } - } - // If search radius + epsilon is not exceeded, request SVS BatchIterator for the next - // batch - if (keep_searching) { - svs_it.next(batch_size, cancel); - if (cancel()) { - rep->code = VecSim_QueryReply_TimedOut; - return rep; + // If search radius + epsilon is not exceeded, request SVS BatchIterator for the + // next batch + if (keep_searching) { + svs_it.next(batch_size, cancel); + if (cancel()) { + rep->code = VecSim_QueryReply_TimedOut; + return rep; + } } } } @@ -719,12 +806,14 @@ class SVSIndex : public VecSimIndexAbstract, fl // take ownership of the blob copy and pass it to the batch iterator. auto *queryBlobCopyPtr = queryBlobCopy.release(); // Ownership of queryBlobCopy moves to VecSimBatchIterator that will free it at the end. - if (indexLabelCount() == 0) { + auto impl = getImpl(); + if (!impl || labelCountOf(*impl) == 0) { return new (this->getAllocator()) NullSVS_BatchIterator(queryBlobCopyPtr, queryParams, this->getAllocator()); } else { return new (this->getAllocator()) SVS_BatchIterator( - queryBlobCopyPtr, impl_.get(), queryParams, this->getAllocator(), is_two_level_lvq); + queryBlobCopyPtr, std::move(impl), queryParams, this->getAllocator(), + is_two_level_lvq); } } @@ -758,15 +847,12 @@ class SVSIndex : public VecSimIndexAbstract, fl } void runGC() override { - if (impl_) { - // There is documentation for consolidate(): - // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.consolidate - impl_->consolidate(); - // There is documentation for compact(): - // https://intel.github.io/ScalableVectorSearch/python/dynamic.html#svs.DynamicVamana.compact - impl_->compact(); + if (auto impl = getImpl()) { + // concurrent compact() consolidates every soft-deleted points first + // pending per-label consolidate jobs need not be drained - they become no-ops + impl->compact(); } - num_marked_deleted = 0; + num_marked_deleted.store(0, std::memory_order_relaxed); } private: @@ -776,7 +862,11 @@ class SVSIndex : public VecSimIndexAbstract, fl template void appendStoredDataByLabel(labelType label, std::vector> &vectors_output) const { - if (!impl_) { + // The spans handed to `append_datum` point straight into the dataset, so the instance has + // to stay alive for the whole walk. The snapshot's reference keeps it alive even if + // markIndexUpdate() drops impl_ meanwhile. + auto impl = getImpl(); + if (!impl) { return; } auto append_datum = [&](auto indexed_span) { @@ -796,18 +886,18 @@ class SVSIndex : public VecSimIndexAbstract, fl }; if constexpr (isMulti) { - auto it = impl_->get_label_to_external_lookup().find(label); - if (it == impl_->get_label_to_external_lookup().end()) { + auto it = impl->get_label_to_external_lookup().find(label); + if (it == impl->get_label_to_external_lookup().end()) { return; } for (auto external_id : it->second) { - append_datum(impl_->get_parent_index().get_datum(external_id)); + append_datum(impl->get_parent_index().get_datum(external_id)); } } else { - if (!impl_->has_id(label)) { + if (!impl->has_id(label)) { return; } - append_datum(impl_->get_datum(label)); + append_datum(impl->get_datum(label)); } } diff --git a/src/VecSim/algorithms/svs/svs_batch_iterator.h b/src/VecSim/algorithms/svs/svs_batch_iterator.h index 116546150..a31c02397 100644 --- a/src/VecSim/algorithms/svs/svs_batch_iterator.h +++ b/src/VecSim/algorithms/svs/svs_batch_iterator.h @@ -29,6 +29,9 @@ class SVS_BatchIterator : public VecSimBatchIterator { using dist_type = typename Index::distance_type; size_t dim; + // Declared before impl_ so that impl_, which holds a raw back-pointer into the index, is + // destroyed first. + std::shared_ptr index_; impl_type impl_; typename impl_type::const_iterator curr_it; size_t batch_size; @@ -67,14 +70,16 @@ class SVS_BatchIterator : public VecSimBatchIterator { } public: - SVS_BatchIterator(void *query_vector, const Index *index, const VecSimQueryParams *queryParams, + SVS_BatchIterator(void *query_vector, std::shared_ptr index, + const VecSimQueryParams *queryParams, std::shared_ptr allocator, bool is_two_level_lvq) : VecSimBatchIterator{query_vector, queryParams ? queryParams->timeoutCtx : nullptr, std::move(allocator)}, - dim{index->dimensions()}, impl_{index->make_batch_iterator(std::span{ - static_cast(query_vector), dim})}, + dim{index->dimensions()}, index_{std::move(index)}, + impl_{index_->make_batch_iterator( + std::span{static_cast(query_vector), dim})}, curr_it{impl_.begin()} { - auto sp = svs_details::joinSearchParams(index->get_search_parameters(), queryParams, + auto sp = svs_details::joinSearchParams(index_->get_search_parameters(), queryParams, is_two_level_lvq); batch_size = queryParams && queryParams->batchSize ? queryParams->batchSize diff --git a/src/VecSim/algorithms/svs/svs_extensions.h b/src/VecSim/algorithms/svs/svs_extensions.h index 2e93a6a58..f1734c79f 100644 --- a/src/VecSim/algorithms/svs/svs_extensions.h +++ b/src/VecSim/algorithms/svs/svs_extensions.h @@ -10,10 +10,20 @@ #pragma once #include "VecSim/algorithms/svs/svs_utils.h" #include "svs/extensions/vamana/scalar.h" +// Tells SQDataset that the concurrent SegmentedBlocked tag is a blocked allocator, so +// that resize()/compact() are not constrained away. Without it a scalar-quantized +// concurrent index fails to compile the moment the index grows. +#include "svs/concurrent/extensions/scalar.h" #if HAVE_SVS_LVQ #include SVS_LVQ_HEADER #include SVS_LEANVEC_HEADER +// The same treatment for the LVQ and LeanVec datasets, each of which keeps its own private +// copy of the blocked-allocator trait. LeanVec additionally picks the allocator for its inner +// datasets through that trait, and its blocked branch hardcodes svs::data::Blocked -- so +// without this the inner storage would quietly lose the grow-stable guarantee. +#include "svs/concurrent/extensions/lvq.h" +#include "svs/concurrent/extensions/leanvec.h" #endif // HAVE_SVS_LVQ // Scalar Quantization traits for SVS @@ -21,7 +31,7 @@ template struct SVSStorageTraits { using element_type = std::int8_t; using allocator_type = svs_details::SVSAllocator; - using blocked_type = svs::data::Blocked>; + using blocked_type = svs::concurrent::SegmentedBlocked>; using index_storage_type = svs::quantization::scalar::SQDataset; @@ -32,7 +42,7 @@ struct SVSStorageTraits { // SVS block size is a power of two, so we can use it directly auto svs_bs = svs_details::SVSBlockSize(block_size, element_size(dim)); allocator_type data_allocator{std::move(allocator)}; - return svs::make_blocked_allocator_handle({svs_bs}, data_allocator); + return svs_details::make_segmented_blocked_allocator_handle({svs_bs}, data_allocator); } static constexpr VecSimSvsQuantBits get_compression_mode() { return VecSimSvsQuant_Scalar; } @@ -89,7 +99,7 @@ template struct SVSStorageTraits 1)>> { using allocator_type = svs_details::SVSAllocator; - using blocked_type = svs::data::Blocked>; + using blocked_type = svs::concurrent::SegmentedBlocked>; using strategy_type = typename svs_details::LVQSelector::strategy; using index_storage_type = svs::quantization::lvq::LVQDataset @@ -181,7 +191,7 @@ struct SVSStorageTraits struct SVSStorageTraits { using allocator_type = svs_details::SVSAllocator; - using blocked_type = svs::data::Blocked>; + using blocked_type = svs::concurrent::SegmentedBlocked>; using index_storage_type = svs::leanvec::LeanDataset, svs::leanvec::UsingLVQ, svs::Dynamic, svs::Dynamic, blocked_type>; @@ -212,7 +222,7 @@ struct SVSStorageTraits { auto elem_size = std::max(primary_element_size(dim), secondary_element_size(dim)); auto svs_bs = svs_details::SVSBlockSize(block_size, elem_size); allocator_type data_allocator{std::move(allocator)}; - return svs::make_blocked_allocator_handle({svs_bs}, data_allocator); + return svs_details::make_segmented_blocked_allocator_handle({svs_bs}, data_allocator); } template diff --git a/src/VecSim/algorithms/svs/svs_serializer_impl.h b/src/VecSim/algorithms/svs/svs_serializer_impl.h index 2780d3457..33fc6ca53 100644 --- a/src/VecSim/algorithms/svs/svs_serializer_impl.h +++ b/src/VecSim/algorithms/svs/svs_serializer_impl.h @@ -76,7 +76,11 @@ template void SVSIndex::impl_save( const std::string &location) { - impl_->save(location + "/config", location + "/graph", location + "/data"); + auto impl = getImpl(); + if (!impl) { + throw std::runtime_error("SVSIndex::impl_save called on an empty index"); + } + impl->save(location + "/config", location + "/graph", location + "/data"); } // This function will load the serialized svs index from the given folder path @@ -92,26 +96,31 @@ void SVSIndex // Verify metadata compatibility, will throw runtime exception if not compatible compareMetadataFile(folder_path + "/metadata"); + std::shared_ptr loaded_impl; if constexpr (isMulti) { - auto loaded = svs::index::vamana::auto_multi_dynamic_assemble( + auto loaded = svs::concurrent::auto_multi_dynamic_assemble( folder_path + "/config", SVS_LAZY(graph_builder_t::load(folder_path + "/graph", this->blockSize, this->buildParams, this->getAllocator())), SVS_LAZY(storage_traits_t::load(folder_path + "/data", this->blockSize, this->dim, this->getAllocator())), distance_f(), std::move(threadpool_handle), - svs::index::vamana::MultiMutableVamanaLoad::FROM_MULTI, logger_); - impl_ = std::make_unique(std::move(loaded)); + svs::concurrent::MultiMutableVamanaLoad::FROM_MULTI, logger_); + loaded_impl = std::make_shared(std::move(loaded)); } else { - auto loaded = svs::index::vamana::auto_dynamic_assemble( + auto loaded = svs::concurrent::auto_dynamic_assemble( folder_path + "/config", SVS_LAZY(graph_builder_t::load(folder_path + "/graph", this->blockSize, this->buildParams, this->getAllocator())), SVS_LAZY(storage_traits_t::load(folder_path + "/data", this->blockSize, this->dim, this->getAllocator())), distance_f(), std::move(threadpool_handle), false, logger_); - impl_ = std::make_unique(std::move(loaded)); + loaded_impl = std::make_shared(std::move(loaded)); } + + std::lock_guard replace_lock(this->implMutationGuard_); + std::lock_guard lock(this->pimplGuard_); + impl_ = std::move(loaded_impl); } template bool SVSIndex::checkIntegrity() const { - if (!impl_) { + auto impl = getImpl(); + if (!impl) { throw std::runtime_error( "SVSIndex integrity check failed: index implementation (impl_) is null."); } @@ -170,9 +180,9 @@ bool SVSIndex try { // SVS internal index integrity validation if constexpr (isMulti) { - impl_->get_parent_index().debug_check_invariants(true); + impl->get_parent_index().debug_check_invariants(true); } else { - impl_->debug_check_invariants(true); + impl->debug_check_invariants(true); } } // debug_check_invariants throws svs::lib::ANNException : public std::runtime_error in case of @@ -182,10 +192,10 @@ bool SVSIndex } try { - size_t index_size = impl_->size(); - size_t storage_size = impl_->view_data().size(); - size_t capacity = storage_traits_t::storage_capacity(impl_->view_data()); - size_t label_count = this->indexLabelCount(); + size_t index_size = impl->size(); + size_t storage_size = impl->view_data().size(); + size_t capacity = storage_traits_t::storage_capacity(impl->view_data()); + size_t label_count = labelCountOf(*impl); // Storage size must match index size if (storage_size != index_size) { @@ -203,7 +213,7 @@ bool SVSIndex bool label_validation_passed = true; try { - impl_->on_ids([&](size_t label) { labels_counted++; }); + impl->on_ids([&](size_t label) { labels_counted++; }); // Validate label count consistency label_validation_passed = (labels_counted == label_count); diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index ca3d019c6..3d7c1fa75 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -11,8 +11,38 @@ #include #include #include +#include +#include #include +/** + * Definition of a job that inserts a new vector from flat into SVS Index. + */ +struct SVSInsertJob : public TieredInsertJob { + // Pending -> Executing -> Done. Delayed means a relabel is remapping the label in the backend + // right now, and publishing would put the vector under a label that is not settled yet. + // Written under flatIndexGuard, except Executing/Done which waitForJob() reads unlocked. + enum class Status : uint8_t { Pending, Delayed, Executing, Done }; + std::atomic status{Status::Pending}; + + SVSInsertJob(std::shared_ptr allocator, labelType label_, idType id_, + JobCallback insertCb, VecSimIndex *index_) + : TieredInsertJob(allocator, SVS_INSERT_VECTOR_JOB, label_, id_, insertCb, index_) {} +}; + +/** + * Definition of a job that launches partial consolidation on SVS Index. + */ +struct SVSConsolidateJob : public AsyncJob { + std::vector labels; + std::atomic executing{false}; + + SVSConsolidateJob(std::shared_ptr allocator, + const std::vector &labels_, JobCallback insertCb, + VecSimIndex *index_) + : AsyncJob(allocator, SVS_CONSOLIDATE_JOB, insertCb, index_), labels(labels_) {} +}; + /** * @class SVSMultiThreadJob * @brief Represents a multi-threaded asynchronous job for the SVS algorithm. @@ -219,29 +249,13 @@ class SVSMultiThreadJob : public AsyncJob { template class TieredSVSIndex : public VecSimTieredIndex { + using DistType = float; using Self = TieredSVSIndex; - using Base = VecSimTieredIndex; - using flat_index_t = BruteForceIndex; - using backend_index_t = VecSimIndexAbstract; + using Base = VecSimTieredIndex; + using flat_index_t = BruteForceIndex; + using backend_index_t = VecSimIndexAbstract; using svs_index_t = SVSIndexBase; - // swaps_journal is used by updateSVSIndex() to track vectors swap operations that were done in - // the Flat index during SVS index updating. - // The journal contains tuples of (label, oldId, newId). - // oldId is the index of the label in flat index before the swap. - // newId is the index of the label in flat index after the swap. - // if oldId == newId, it means that the vector was not moved in the Flat index, but was removed - // from the end of the Flat index (no id swaps occurred internally). - // if label == SKIP_LABEL, it means that the vector was not moved in the Flat index, but - // updated in-place (hence was already removed and no need to remove it again) - using swap_record = std::tuple; - constexpr static size_t SKIP_LABEL = std::numeric_limits::max(); - std::vector swaps_journal; - // deleted_labels_journal is used by updateSVSIndex() to track vectors that were deleted from - // Flat index during SVS index updating. The journal contains the deleted labels. These labels - // are used to delete the same vectors from the SVS index at the end of the update. - std::vector deleted_labels_journal; - size_t trainingTriggerThreshold; size_t updateTriggerThreshold; size_t updateJobWaitTime; @@ -252,11 +266,20 @@ class TieredSVSIndex : public VecSimTieredIndex { std::atomic_flag indexGCScheduled = ATOMIC_FLAG_INIT; // Used to prevent running multiple index update jobs in parallel. // Even if update jobs scheduled sequentially, they can be started in parallel. - mutable std::mutex updateJobMutex; + mutable std::shared_mutex updateJobMutex; // The reason of following container just to properly destroy jobs which not executed yet SVSMultiThreadJob::JobsRegistry uncompletedJobs; + // frontend ids holding training data for the backend (re)initialization + std::unordered_set ids_to_init_; + + vecsim_stl::unordered_map> + labelToConsolidateJobs; + mutable std::shared_mutex consolidateJobsGuard; + + size_t flat_buffer_bound; + /// //////////////////////////////////////////////////////////////////////////////////////////////////// // TieredSVS_BatchIterator // @@ -280,21 +303,15 @@ class TieredSVSIndex : public VecSimTieredIndex { VecSimBatchIterator *flat_iterator; VecSimBatchIterator *svs_iterator; - std::shared_lock svs_lock; - - // On single value indices, this set holds the IDs of the results that were returned from - // the flat buffer. - // On multi value indices, this set holds the IDs of all the results that were returned. - // The difference between the two cases is that on multi value indices, the same ID can - // appear in both indexes and results with different scores, and therefore we can't tell in - // advance when we expect a possibility of a duplicate. - // On single value indices, a duplicate may appear at the same batch (and we will handle it - // when merging the results) Or it may appear in a different batches, first from the flat - // buffer and then from the SVS, in the cases where a better result if found later in SVS - // because of the approximate nature of the algorithm. + + // IDs of all the results returned so far, so that neither tier can yield one twice. + // Both directions have to be covered: nothing freezes the backend for the iterator's + // lifetime, so an insert job can publish a label into SVS and only then drop it from the + // flat buffer, while the flat iterator still holds it in the snapshot it computed on its + // first call. Either tier can therefore surface a label first. vecsim_stl::unordered_set returned_results_set; - VecSimQueryReply *compute_current_batch(size_t n_res, bool isMultiValue) { + VecSimQueryReply *compute_current_batch(size_t n_res) { // Merge results // This call will update `svs_res` and `bf_res` to point to the end of the merged // results. @@ -304,33 +321,18 @@ class TieredSVSIndex : public VecSimTieredIndex { auto [from_svs, from_flat] = merge_results(batch_res->results, svs_results, flat_results, n_res); - if (!isMultiValue) { - // If we're on a single-value index, update the set of results returned from the - // FLAT index before popping them, to prevent them to be returned from the SVS index - // in later batches. - for (size_t i = 0; i < from_flat; ++i) { - this->returned_results_set.insert(this->flat_results[i].id); - } - } else { - // If we're on a multi-value index, update the set of results returned (from - // `batch_res`) - for (size_t i = 0; i < batch_res->results.size(); ++i) { - this->returned_results_set.insert(batch_res->results[i].id); - } + for (size_t i = 0; i < batch_res->results.size(); ++i) { + this->returned_results_set.insert(batch_res->results[i].id); } // Update results flat_results.erase(flat_results.begin(), flat_results.begin() + from_flat); svs_results.erase(svs_results.begin(), svs_results.begin() + from_svs); - // clean up the results - // On multi-value indexes, one (or both) results lists may contain results that are - // already returned form the other list (with a different score). We need to filter them - // out. - if (isMultiValue) { - filter_irrelevant_results(this->flat_results); - filter_irrelevant_results(this->svs_results); - } + // Either list may still hold a label just returned from the other one (with a different + // score), so drop those before the next batch. + filter_irrelevant_results(this->flat_results); + filter_irrelevant_results(this->svs_results); // Return current batch return batch_res; @@ -346,7 +348,6 @@ class TieredSVSIndex : public VecSimTieredIndex { void acquire_svs_iterator() { assert(svs_iterator == nullptr); - this->index->mainIndexGuard.lock_shared(); svs_iterator = index->backendIndex->newBatchIterator( this->flat_iterator->getQueryBlob(), queryParams); } @@ -355,7 +356,6 @@ class TieredSVSIndex : public VecSimTieredIndex { if (svs_iterator != nullptr && svs_iterator != depleted()) { delete svs_iterator; svs_iterator = nullptr; - this->index->mainIndexGuard.unlock_shared(); } } @@ -380,8 +380,7 @@ class TieredSVSIndex : public VecSimTieredIndex { std::move(allocator)), index(index), flat_results(this->allocator), svs_results(this->allocator), flat_iterator(index->frontendIndex->newBatchIterator(query_vector, queryParams)), - svs_iterator(nullptr), svs_lock(index->mainIndexGuard, std::defer_lock), - returned_results_set(this->allocator) { + svs_iterator(nullptr), returned_results_set(this->allocator) { if (queryParams) { this->queryParams = (VecSimQueryParams *)this->allocator->allocate(sizeof(VecSimQueryParams)); @@ -402,7 +401,6 @@ class TieredSVSIndex : public VecSimTieredIndex { VecSimQueryReply *getNextResults(size_t n_res, VecSimQueryReply_Order order) override { auto svs_code = VecSim_QueryReply_OK; - const bool isMulti = this->index->backendIndex->isMultiValue(); if (svs_iterator == nullptr) { // first call // First call to getNextResults. The call to the BF iterator will include // calculating all the distances and access the BF index. We take the lock on this @@ -433,17 +431,8 @@ class TieredSVSIndex : public VecSimTieredIndex { tail->results.end()); VecSimQueryReply_Free(tail); - if (!isMulti) { - // On single-value indexes, duplicates will never appear in the hnsw results - // before they appear in the flat results (at the same time or later if the - // approximation misses) so we don't need to try and filter the flat results - // (and recheck conditions). - break; - } else { - // On multi-value indexes, the flat results may contain results that are - // already returned from the hnsw index. We need to filter them out. - filter_irrelevant_results(this->flat_results); - } + // The flat results may contain labels already returned from the SVS index. + filter_irrelevant_results(this->flat_results); } while (svs_results.size() < n_res && svs_iterator != depleted() && @@ -469,7 +458,7 @@ class TieredSVSIndex : public VecSimTieredIndex { } VecSimQueryReply *batch; - batch = compute_current_batch(n_res, isMulti); + batch = compute_current_batch(n_res); if (order == BY_ID) { sort_results_by_id(batch); @@ -548,30 +537,61 @@ class TieredSVSIndex : public VecSimTieredIndex { } #endif +protected: + ScopedLocks lockMainIndexForQuery() const override { + // No-op: SVS does its query locking internally. + return ScopedLocks(); + } + + ScopedLocks lockIndexForSize() const override { return ScopedLocks(this->flatIndexLockable); } + + ScopedLocks lockIndexForCapacity() const override { + return ScopedLocks(this->flatIndexLockable); + } + private: /** - * @brief Updates the SVS index in a thread-safe manner. + * @brief Init the SVS index in a thread-safe manner. * * This static wrapper function performs the following actions: * - Acquires a lock on the index's updateJobMutex to prevent concurrent updates. - * - Clears the indexUpdateScheduled flag to allow future scheduling. * - Configures the number of threads for the underlying SVS index update operation. - * - Calls the updateSVSIndex method to perform the actual index update. + * - Calls the initSVSIndex method to perform the actual index update. + * - Clears the indexUpdateScheduled flag to allow future scheduling. * * @param idx Pointer to the VecSimIndex to be updated. * @param availableThreads The number of threads available for the update operation. Current * thread us used as well, so the minimal value is 1. */ - static void updateSVSIndexWrapper(VecSimIndex *idx, size_t availableThreads) { + static void initSVSIndexWrapper(VecSimIndex *idx, size_t availableThreads) { assert(availableThreads > 0); auto index = static_cast *>(idx); assert(index); // prevent parallel updates - std::lock_guard lock(index->updateJobMutex); - // Release the scheduled flag to allow scheduling again - index->indexUpdateScheduled.clear(); - // Update the SVS index - index->updateSVSIndex(availableThreads); + std::lock_guard lock(index->updateJobMutex); + // flag stays set while init runs: !ready() + !flag means "no one will init the backend" + struct ClearOnExit { + std::atomic_flag &flag; + ~ClearOnExit() { flag.clear(std::memory_order_release); } + } clear_on_exit{index->indexUpdateScheduled}; + index->initSVSIndex(availableThreads); + } + + static void executeInsertJobWrapper(AsyncJob *job) { + auto *insert_job = static_cast(job); + auto *job_index = static_cast *>(insert_job->index); + InsertJobOutcome outcome; + { + // prevent parallel execution with index initilizing job + std::shared_lock lock(job_index->updateJobMutex); + outcome = job_index->executeInsertJob(insert_job); + } + + if (outcome == InsertJobOutcome::Deferred) { + job_index->submitSingleJob(job); + return; + } + delete job; } /** @@ -594,35 +614,64 @@ class TieredSVSIndex : public VecSimTieredIndex { auto index = static_cast *>(idx); assert(index); - std::lock_guard lock{index->mainIndexGuard}; - // Release the scheduled flag to allow scheduling again - index->indexGCScheduled.clear(); - // Do SVS index GC index->backendIndex->log(VecSimCommonStrings::LOG_VERBOSE_STRING, "running asynchronous GC for tiered SVS index"); auto svs_index = index->GetSVSIndex(); if (index->backendIndex->indexSize() == 0) { // No need to run GC on an empty index. + index->indexGCScheduled.clear(); return; } index->executeTracingCallback("GCJob::before_run_gc"); + std::lock_guard lock(index->updateJobMutex); + + // Release the scheduled flag to allow scheduling again + index->indexGCScheduled.clear(); + svs_index->setParallelism(std::min(availableThreads, index->backendIndex->indexSize())); // VecSimIndexAbstract::runGC() is protected static_cast(index->backendIndex)->runGC(); + svs_index->setParallelism(1); + } + + static void SVSIndexConsolidateWrapper(AsyncJob *job) { + auto consolidate_job = static_cast(job); + auto index = static_cast *>(consolidate_job->index); + + std::shared_lock lock(index->updateJobMutex); + auto svs_index = index->GetSVSIndex(); + svs_index->setParallelism(1); + + bool valid = false; + { + std::shared_lock flat_lock(index->flatIndexGuard); + valid = consolidate_job->isValid; + if (valid) { + consolidate_job->executing.store(true, std::memory_order_release); + } + } + if (valid) { + svs_index->consolidate(consolidate_job->labels); + // Cleared before the registry lock is needed again, so a waiter cannot deadlock us. + consolidate_job->executing.store(false, std::memory_order_release); + } + index->forgetConsolidateJob(consolidate_job); + delete job; } #ifdef BUILD_TESTS public: #endif - void scheduleSVSIndexUpdate() { + + void scheduleSVSIndexInit() { // do not schedule if scheduled already if (indexUpdateScheduled.test_and_set()) { return; } auto jobs = SVSMultiThreadJob::createScheduledJobs( - this->allocator, SVS_BATCH_UPDATE_JOB, updateSVSIndexWrapper, this, + this->allocator, SVS_BATCH_UPDATE_JOB, initSVSIndexWrapper, this, std::chrono::microseconds(updateJobWaitTime), &uncompletedJobs); this->submitJobs(jobs); } @@ -639,128 +688,211 @@ class TieredSVSIndex : public VecSimTieredIndex { this->submitJobs(jobs); } + void scheduleSVSIndexConsolidate(labelType label) { + auto *new_consolidate_job = new (this->allocator) + SVSConsolidateJob(this->allocator, {label}, SVSIndexConsolidateWrapper, this); + + { + std::lock_guard lock(this->consolidateJobsGuard); + auto it = this->labelToConsolidateJobs.find(label); + if (it != this->labelToConsolidateJobs.end()) { + it->second.push_back(new_consolidate_job); + } else { + vecsim_stl::vector jobs(1, new_consolidate_job, + this->allocator); + this->labelToConsolidateJobs.insert({label, std::move(jobs)}); + } + } + // Insert job to the queue. + this->submitSingleJob(new_consolidate_job); + } + private: - static void applySwapsToLabelsArray(std::vector &labels, - const std::vector &swaps) { - // Enumerate journal and reflect swaps in the labels. - // The journal contains tuples of (label, oldId, newId). - // oldId is the index of the label in flat index before the swap. - // newId is the index of the label in flat index after the swap. - for (const auto &p : swaps) { - auto oldId = std::get<1>(p); - auto newId = std::get<2>(p); - - if (oldId == newId || oldId >= labels.size()) { - // If oldId == newId, it means that the vector was not moved in the Flat index, - // but was removed or updated in-place. - // If oldId is out of bounds - new vector was added and swapped meanwhile. - // In both cases, we should not touch flat index at the new position. - if (newId < labels.size()) { - labels[newId] = SKIP_LABEL; + void forgetConsolidateJob(SVSConsolidateJob *job) { + std::lock_guard lock(this->consolidateJobsGuard); + for (auto label : job->labels) { + auto it = this->labelToConsolidateJobs.find(label); + if (it == this->labelToConsolidateJobs.end()) { + continue; + } + auto &jobs = it->second; + jobs.erase(std::remove(jobs.begin(), jobs.end(), job), jobs.end()); + if (jobs.empty()) { + this->labelToConsolidateJobs.erase(it); + } + } + } + // Caller must hold flatIndexGuard exclusive + std::vector takeOverConsolidateOf(labelType label) { + std::vector taken_over; + vecsim_stl::vector running(this->allocator); + { + std::shared_lock lock(this->consolidateJobsGuard); + auto it = this->labelToConsolidateJobs.find(label); + if (it == this->labelToConsolidateJobs.end()) { + return taken_over; + } + for (auto *job : it->second) { + if (!job->isValid) { + continue; // already taken over by an earlier call + } + if (job->executing.load(std::memory_order_acquire)) { + running.push_back(job); + } else { + job->isValid = false; + taken_over.insert(taken_over.end(), job->labels.begin(), job->labels.end()); } - continue; // Next swap record. } + } + for (auto *job : running) { + waitForJob(job); + } + return taken_over; + } + + // Wait until the job leaves its publish window + // Safe to call while holding flatIndexGuard exclusive + static void waitForJob(SVSInsertJob *job) { + while (job->status.load(std::memory_order_acquire) == SVSInsertJob::Status::Executing) { + std::this_thread::yield(); + } + } + + static void waitForJob(SVSConsolidateJob *job) { + while (job->executing.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + } + + idType setAndSaveInvalidJob(AsyncJob *job) override { + waitForJob(static_cast(job)); + return Base::setAndSaveInvalidJob(job); + } + + enum class InsertJobOutcome { Completed, Deferred }; - // Real swap case. - // If oldId != newId and oldId is in bounds, it means that the tracked vector was moved - // So, move the label to new position and skip the old position from deletions. - // NOTE: labels[oldId] can be SKIP_LABEL, but we still need to move it to newId. - labels[newId] = labels[oldId]; - labels[oldId] = SKIP_LABEL; + InsertJobOutcome executeInsertJob(SVSInsertJob *job) { + auto svs_index = GetSVSIndex(); + + // Note that accessing the job fields should occur with flat index guard held (here and + // later). + this->flatIndexGuard.lock_shared(); + if (!job->isValid) { + this->flatIndexGuard.unlock_shared(); + // Job has been invalidated in the meantime - nothing to execute, and remove it from the + // lookup. + this->invalidJobsLookupGuard.lock(); + this->invalidJobs.erase(job->id); + this->invalidJobsLookupGuard.unlock(); + return InsertJobOutcome::Completed; + } + + if (job->status.load(std::memory_order_relaxed) == SVSInsertJob::Status::Delayed) { + this->flatIndexGuard.unlock_shared(); + // relabelVector() clears it once the backend remap settled the label + return InsertJobOutcome::Deferred; + } + + // a job never inits the backend: one point cannot train the compression + if (!svs_index->ready()) { + this->flatIndexGuard.unlock_shared(); + // init pending -> wait for it. no init pending -> back to the training buffer + return this->indexUpdateScheduled.test(std::memory_order_acquire) + ? InsertJobOutcome::Deferred + : adoptJobIntoInitBuffer(job); } + + // Copy the vector blob out of the flat buffer while holding flatIndexGuard, so we + // can release the flat lock before indexing into the SVS backend + size_t data_size = this->frontendIndex->getStoredDataSize(); + auto blob_copy = this->getAllocator()->allocate_unique(data_size); + memcpy(blob_copy.get(), this->frontendIndex->getDataByInternalId(job->id), data_size); + + job->status.store(SVSInsertJob::Status::Executing, std::memory_order_release); + this->flatIndexGuard.unlock_shared(); + + // if a concurrent deletion drops the instance + // initializing it from this single point would refit the compression + const int added = svs_index->addVectorsIfInitialized(blob_copy.get(), &job->label, 1); + const bool published = added != SVSIndexBase::kNotInitialized; + + job->status.store(published ? SVSInsertJob::Status::Done : SVSInsertJob::Status::Pending, + std::memory_order_release); + if (!published) { + return adoptJobIntoInitBuffer(job); + } + // Remove the vector and the insert job from the flat buffer. + this->removeIngestedVectorFromFlat(job); + return InsertJobOutcome::Completed; } - void updateSVSIndex(size_t availableThreads) { + void initSVSIndex(size_t availableThreads) { + std::vector ids_to_move; std::vector labels_to_move; std::vector vectors_to_move; - std::vector deleted_labels_during_update; + executeTracingCallback("UpdateJob::before_add_to_svs"); { // lock frontendIndex from modifications - std::shared_lock flat_lock{this->flatIndexGuard}; + // The whole initialization is done under flatIndexGuard + std::lock_guard flat_lock{this->flatIndexGuard}; auto flat_index = this->GetFlatIndex(); - const auto frontend_index_size = this->frontendIndex->indexSize(); + const auto init_batch_size = ids_to_init_.size(); const size_t dim = flat_index->getDim(); - labels_to_move.reserve(frontend_index_size); - vectors_to_move.reserve(frontend_index_size * dim); - - for (idType i = 0; i < frontend_index_size; i++) { - labels_to_move.push_back(flat_index->getVectorLabel(i)); - auto data = flat_index->getDataByInternalId(i); + ids_to_move.reserve(init_batch_size); + labels_to_move.reserve(init_batch_size); + vectors_to_move.reserve(init_batch_size * dim); + + for (idType id : ids_to_init_) { + ids_to_move.push_back(id); + labels_to_move.push_back(flat_index->getVectorLabel(id)); + auto data = flat_index->getDataByInternalId(id); vectors_to_move.insert(vectors_to_move.end(), data, data + dim); } - // reset journal to the current frontend index state - swaps_journal.clear(); - deleted_labels_journal.clear(); - } // release frontend index - - executeTracingCallback("UpdateJob::before_add_to_svs"); - if (!labels_to_move.empty()) { - // lock backend index for writing and add vectors there - std::shared_lock main_shared_lock(this->mainIndexGuard); - auto svs_index = GetSVSIndex(); - assert(labels_to_move.size() == vectors_to_move.size() / this->frontendIndex->getDim()); - if (this->backendIndex->indexSize() == 0) { - // If backend index is empty, we need to initialize it first. - svs_index->setParallelism(std::min(availableThreads, labels_to_move.size())); - auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), - labels_to_move.size()); + ids_to_init_.clear(); - // Upgrade to unique lock to set the new impl - main_shared_lock.unlock(); - std::lock_guard lock(this->mainIndexGuard); - svs_index->setImpl(std::move(impl)); - } else { - // Backend index is initialized - just add the vectors. - main_shared_lock.unlock(); - std::lock_guard lock(this->mainIndexGuard); - // Upgrade to unique lock to add vectors + // Nothing to initialize from an empty batch, and setParallelism(0) is not a + // valid request against the shared pool. + if (!labels_to_move.empty()) { + auto svs_index = GetSVSIndex(); svs_index->setParallelism(std::min(availableThreads, labels_to_move.size())); - svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), - labels_to_move.size()); + assert(labels_to_move.size() == + vectors_to_move.size() / this->frontendIndex->getDim()); + // The backend may already have been initialized while this job was queued + if (svs_index->ready()) { + svs_index->addVectors(vectors_to_move.data(), labels_to_move.data(), + labels_to_move.size()); + svs_index->setParallelism(1); + } else { + auto impl = svs_index->createImpl(vectors_to_move.data(), labels_to_move.data(), + labels_to_move.size()); + svs_index->setParallelism(1); + svs_index->setImpl(std::move(impl)); + } } - } - executeTracingCallback("UpdateJob::after_add_to_svs"); - // clean-up frontend index - { // lock frontend index for writing and delete moved vectors - std::lock_guard lock(this->flatIndexGuard); - - // swap deleted labels journal with the local variable to track deleted labels during - // update - std::swap(deleted_labels_during_update, deleted_labels_journal); - // Apply swaps from journal to labels_to_move to reflect changes made in meanwhile. - applySwapsToLabelsArray(labels_to_move, this->swaps_journal); - - // delete vectors from the frontend index in reverse order - // it increases the chance of avoiding swaps in the frontend index and performance - // improvement - int deleted = 0; - idType id = labels_to_move.size(); - while (id-- > 0) { - auto label = labels_to_move[id]; + std::sort(ids_to_move.begin(), ids_to_move.end()); + [[maybe_unused]] int total_deleted = 0; + for (auto it = ids_to_move.rbegin(); it != ids_to_move.rend(); ++it) { + idType id = *it; + auto label = this->frontendIndex->getVectorLabel(id); // Delete the vector from the frontend index if not in-place updated. - if (label != SKIP_LABEL) { - deleted += this->frontendIndex->deleteVectorById(label, id); + labelType last_vec_label = + this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); + int deleted = this->frontendIndex->deleteVectorById(label, id); + if (deleted && id != this->frontendIndex->indexSize()) { + // If the vector removal caused a swap with the last id, update the relevant + // insert job. + this->updateInsertJobInternalId(this->frontendIndex->indexSize(), id, + last_vec_label); } + total_deleted += deleted; } - assert(deleted == std::count_if(labels_to_move.begin(), labels_to_move.end(), - [](labelType label) { return label != SKIP_LABEL; }) && - "Deleted vectors count does not match the number of labels to delete"); - } - // delete vectors from backend index that were deleted from the frontend index during - // the update process. - { - std::lock_guard main_lock(this->mainIndexGuard); - std::sort(deleted_labels_during_update.begin(), deleted_labels_during_update.end()); - auto it = std::unique(deleted_labels_during_update.begin(), - deleted_labels_during_update.end()); - deleted_labels_during_update.erase(it, deleted_labels_during_update.end()); - auto svs_index = GetSVSIndex(); - svs_index->deleteVectors(deleted_labels_during_update.data(), - deleted_labels_during_update.size()); - } + assert(total_deleted == labels_to_move.size() && + "Deleted vectors count does not match the number of labels to delete"); + } // release frontend index + executeTracingCallback("UpdateJob::after_add_to_svs"); } public: @@ -768,13 +900,13 @@ class TieredSVSIndex : public VecSimTieredIndex { const TieredIndexParams &tiered_index_params, std::shared_ptr allocator) : Base(svs_index, bf_index, tiered_index_params, allocator), - uncompletedJobs(this->allocator) { + uncompletedJobs(this->allocator), labelToConsolidateJobs(this->allocator) { const auto &tiered_svs_params = tiered_index_params.specificParams.tieredSVSParams; // If flatBufferLimit is not initialized (0), use the default update threshold. - const size_t flat_buffer_bound = tiered_index_params.flatBufferLimit == 0 - ? SVS_VAMANA_DEFAULT_UPDATE_THRESHOLD - : tiered_index_params.flatBufferLimit; + flat_buffer_bound = tiered_index_params.flatBufferLimit == 0 + ? SVS_VAMANA_DEFAULT_UPDATE_THRESHOLD + : tiered_index_params.flatBufferLimit; this->updateTriggerThreshold = tiered_svs_params.updateTriggerThreshold == 0 @@ -794,176 +926,315 @@ class TieredSVSIndex : public VecSimTieredIndex { this->updateJobWaitTime = tiered_svs_params.updateJobWaitTime == 0 ? SVS_DEFAULT_UPDATE_JOB_WAIT_TIME : tiered_svs_params.updateJobWaitTime; + } - // Reserve space for the journal to avoid reallocation. - this->swaps_journal.reserve(this->trainingTriggerThreshold); + ~TieredSVSIndex() { + // Delete all the pending consolidate jobs + for (auto &jobs : this->labelToConsolidateJobs) { + for (auto *job : jobs.second) { + delete job; + } + } } int addVector(const void *blob, labelType label) override { int ret = 0; auto svs_index = GetSVSIndex(); - size_t update_threshold = 0; - size_t frontend_index_size = 0; // In-Place mode - add vector syncronously to the backend index. if (this->getWriteMode() == VecSim_WriteInPlace) { - // It is ok to lock everything at once for in-place mode, - // but we will have to unlock averything before calling updateSVSIndexWrapper() - // so make the minimal needed lock here. - std::shared_lock backend_shared_lock(this->mainIndexGuard); // Backend index initialization data have to be buffered for proper // compression/training. - if (this->backendIndex->indexSize() == 0) { - // If backend index size is 0, first collect vectors in frontend index - // lock in scope to ensure that these will be released before - // updateSVSIndexWrapper() is called. - { - std::lock_guard lock(this->flatIndexGuard); - ret = this->frontendIndex->addVector(blob, label); - // If frontend size exceeds the update job threshold, ... - frontend_index_size = this->frontendIndex->indexSize(); + if (!svs_index->ready()) { + if (auto buffered = tryBufferForTraining(blob, label, /*in_place=*/true)) { + return *buffered; } - // ... move vectors to the backend index. - if (frontend_index_size >= this->trainingTriggerThreshold) { - // updateSVSIndexWrapper() accures it's own locks - backend_shared_lock.unlock(); - // initialize the SVS index synchonously using current thread only - updateSVSIndexWrapper(this, 1); - } - return ret; - } else { - // backend index is initialized - we can add the vector directly - backend_shared_lock.unlock(); - auto storage_blob = this->frontendIndex->preprocessForStorage(blob); + } + // backend index is initialized - we can add the vector directly + auto storage_blob = this->frontendIndex->preprocessForStorage(blob); + int deleted = 0; + { // prevent update job from running in parallel and lock any access to the backend // index - std::scoped_lock lock(this->updateJobMutex, this->mainIndexGuard); + // Only updateJobMutex is needed here, not mainIndexGuard: the concurrent + // backend index serializes writes against concurrent readers itself. + std::lock_guard lock(this->updateJobMutex); // Defensive: ensure single-threaded operation for write-in-place mode. // parallelism_ defaults to 1, so this is a no-op in the normal case. svs_index->setParallelism(1); - return this->backendIndex->addVector(storage_blob.get(), label); + if (!this->backendIndex->isMultiValue()) { + deleted = svs_index->deleteVector(label); + if (deleted > 0) + svs_index->consolidate({label}); + } + if (svs_index->ready()) { + return this->backendIndex->addVector(storage_blob.get(), label) - deleted; + } } + // the delete above dropped the last backend vector: adding now would refit the + // compression to this single point + if (auto buffered = tryBufferForTraining(blob, label, /*in_place=*/true)) { + return std::max(*buffered - deleted, 0); + } + // a queued init job rebuilt the backend meanwhile + std::lock_guard lock(this->updateJobMutex); + return std::max(this->backendIndex->addVector(storage_blob.get(), label) - deleted, 0); } assert(this->getWriteMode() != VecSim_WriteInPlace && "InPlace mode returns early"); - // Async mode - add vector to the frontend index and schedule an update job if needed. - if (!this->backendIndex->isMultiValue()) { - { - std::shared_lock lock(this->flatIndexGuard); - // If the label already exists in the frontend index, we should count it - // to prevent the case when existing vector is moved meanwhile by the update job. - if (this->frontendIndex->isLabelExists(label)) { - ret = -1; - } - } - // Remove vector from the backend index if it exists in case of non-MULTI. - auto label_exists = [&]() { - std::shared_lock lock(this->mainIndexGuard); - return svs_index->isLabelExists(label); - }(); - - if (label_exists) { - std::lock_guard lock(this->mainIndexGuard); - ret -= this->backendIndex->deleteVector(label); + // Async mode - buffer training data until the backend can be inited from a batch. + if (!svs_index->ready() && !this->indexUpdateScheduled.test(std::memory_order_acquire)) { + if (auto buffered = tryBufferForTraining(blob, label, /*in_place=*/false)) { + return *buffered; } } - { // Add vector to the frontend index. - std::lock_guard lock(this->flatIndexGuard); - const auto ft_ret = this->frontendIndex->addVector(blob, label); - if (ft_ret == 0) { // Vector was overriden - add 'skiping' swap to the journal. - assert(!this->backendIndex->isMultiValue() && - "addVector() may return 0 for single value indices only"); - for (auto id : this->frontendIndex->getElementIds(label)) { - this->swaps_journal.emplace_back(SKIP_LABEL, id, id); + this->flatIndexGuard.lock_shared(); + if (svs_index->ready() && (this->frontendIndex->indexSize() >= flat_buffer_bound)) { + this->flatIndexGuard.unlock_shared(); + auto storage_blob = this->frontendIndex->preprocessForStorage(blob); + + const int overwritten = + this->backendIndex->isMultiValue() ? 0 : this->deleteVector(label); + + // the delete above may have dropped the last backend vector + if (!svs_index->ready()) { + if (auto buffered = tryBufferForTraining(blob, label, /*in_place=*/false)) { + return std::max(*buffered - overwritten, 0); } - deleted_labels_journal.push_back(label); } - ret = std::max(ret + ft_ret, 0); - // Check frontend index size to determine if an update job schedule is needed. - frontend_index_size = this->frontendIndex->indexSize(); - } - { - // If main index is empty then update_threshold is trainingTriggerThreshold, - // overwise it is updateTriggerThreshold. - std::shared_lock lock(this->mainIndexGuard); - update_threshold = this->backendIndex->indexSize() == 0 ? this->trainingTriggerThreshold - : this->updateTriggerThreshold; - } - if (frontend_index_size >= update_threshold) { - scheduleSVSIndexUpdate(); + + std::shared_lock lock(updateJobMutex); + ret = svs_index->addVector(storage_blob.get(), label); + return std::max(ret - overwritten, 0); + } else { + this->flatIndexGuard.unlock_shared(); + this->flatIndexGuard.lock(); + idType new_flat_id = this->frontendIndex->indexSize(); + if (this->frontendIndex->isLabelExists(label) && !this->frontendIndex->isMultiValue()) { + if (this->labelToInsertJobs.count(label) == 0) { + // No pending insert job. + // Just replace vector in the frontend + // If this label already exists, this will do overwrite. + ret = this->frontendIndex->addVector(blob, label); + this->flatIndexGuard.unlock(); + return ret; + } + + // Overwrite the vector and invalidate its only pending job (since we are not in + // MULTI). Label exists, but job doesn't. It means the label is used for + // initialization. Just create new job, this job will remove duplicate from the + // backend. + auto *old_job = this->labelToInsertJobs.at(label).at(0); + old_job->id = this->setAndSaveInvalidJob(old_job); + this->labelToInsertJobs.erase(label); + ret = 0; + // We are going to update the internal id that currently holds the vector associated + // with the given label. + new_flat_id = + dynamic_cast *>(this->frontendIndex) + ->getIdOfLabel(label); + // If we are adding a new element (rather than updating an exiting one) we may need + // to increase index capacity. + } + // If this label already exists, this will do overwrite. + ret += this->frontendIndex->addVector(blob, label); + + TieredInsertJob *new_insert_job = createInsertJob(label, new_flat_id); + this->flatIndexGuard.unlock(); + + // Here, a worker might ingest the previous vector that was stored under "label" + // (in case of override in non-MULTI index) - so if it's there, we remove it + // we submit the insert job. + if (!this->backendIndex->isMultiValue()) { + if (svs_index->ready()) { + // If we removed the previous vector from both svs and flat in the overwrite + // process, we still return 0 (not -1). + auto deleted = svs_index->deleteVector(label); + if (deleted > 0) + scheduleSVSIndexConsolidate(label); + ret = std::max(ret - deleted, 0); + } + } + + // Insert job to the queue and signal the workers' updater. + this->submitSingleJob(new_insert_job); + return ret; } + } - return ret; + // training data is tracked by frontend id, so it has to follow the same swaps as insert jobs + void updateInsertJobInternalId(idType prev_id, idType new_id, labelType label) override { + if (ids_to_init_.erase(prev_id) > 0) { + ids_to_init_.insert(new_id); + } + Base::updateInsertJobInternalId(prev_id, new_id, label); } - int deleteAndRecordSwaps_Unsafe(labelType label) { + // Returns the number of vectors removed from the frontend index. + int deleteAndUpdateInitIds(labelType label) { auto deleting_ids = this->frontendIndex->getElementIds(label); - // assert if all elements of deleting_ids are unique - assert(std::set(deleting_ids.begin(), deleting_ids.end()).size() == deleting_ids.size() && - "deleting_ids should contain unique ids"); - - // Sort deleting_ids by id descending order - std::sort(deleting_ids.begin(), deleting_ids.end(), - [](const auto &a, const auto &b) { return a > b; }); + std::sort(deleting_ids.begin(), deleting_ids.end()); // Delete vector from the frontend index. auto updated_ids = this->frontendIndex->deleteVectorAndGetUpdatedIds(label); assert(std::all_of(updated_ids.begin(), updated_ids.end(), [&deleting_ids](const auto &pair) { - return std::find(deleting_ids.begin(), deleting_ids.end(), - pair.first) != deleting_ids.end(); + return std::binary_search(deleting_ids.begin(), deleting_ids.end(), + pair.first); }) && "updated_ids should be a subset of deleting_ids"); - // Record swaps in the journal. - for (auto id : deleting_ids) { - auto it = updated_ids.find(id); - if (it != updated_ids.end()) { - assert(id == it->first && "id in updated_ids should match the id in deleting_ids"); - auto newId = id; - auto oldId = it->second.first; - auto oldLabel = it->second.second; - this->swaps_journal.emplace_back(oldLabel, oldId, newId); - } else { - // No swap, just delete is marked by oldId == newId == deleted id - this->swaps_journal.emplace_back(SKIP_LABEL, id, id); + std::vector new_ids_to_init; + for (auto &it : updated_ids) { + idType prev_id = it.second.first; + idType new_id = it.first; + + if (ids_to_init_.count(prev_id) > 0) { + ids_to_init_.erase(prev_id); + new_ids_to_init.push_back(new_id); + } + labelType updated_vec_label = it.second.second; + this->updateInsertJobInternalId(prev_id, new_id, updated_vec_label); + } + + for (idType new_id : new_ids_to_init) { + ids_to_init_.insert(new_id); + } + + return static_cast(deleting_ids.size()); + } + + int removeLabelFromFlat(labelType label) { + if (!this->frontendIndex->isLabelExists(label)) { + return 0; + } + auto deleting_ids = this->frontendIndex->getElementIds(label); + if (deleting_ids.size() == 0) { + return 0; + } + + // assert if all elements of deleting_ids are unique + assert(std::set(deleting_ids.begin(), deleting_ids.end()).size() == deleting_ids.size() && + "deleting_ids should contain unique ids"); + + // If id is deleted, don't use it for initialization + if (!ids_to_init_.empty()) { + for (idType id : deleting_ids) { + ids_to_init_.erase(id); + } + } + + if (this->labelToInsertJobs.count(label) > 0) { + // Invalidate the pending insert job(s) into SVS associated with this label + auto &insert_jobs = this->labelToInsertJobs.at(label); + for (auto *job : insert_jobs) { + job->id = this->setAndSaveInvalidJob(job); } + // Remove the pending insert job(s) from the labelToInsertJobs mapping. + this->labelToInsertJobs.erase(label); } - deleted_labels_journal.push_back(label); - return deleting_ids.size(); + deleteAndUpdateInitIds(label); + return static_cast(deleting_ids.size()); } - int deleteVector(labelType label) override { + // Create a pending insert job for a vector already buffered in the frontend index + // Caller must hold flatIndexGuard exclusively. + TieredInsertJob *createInsertJob(labelType label, idType flat_id) { + TieredInsertJob *job = new (this->allocator) + SVSInsertJob(this->allocator, label, flat_id, executeInsertJobWrapper, this); + auto it = this->labelToInsertJobs.find(label); + if (it != this->labelToInsertJobs.end()) { + // There's already a pending insert job for this label, add another one (without + // overwrite, only possible in multi index) + assert(this->backendIndex->isMultiValue()); + it->second.push_back(job); + } else { + vecsim_stl::vector new_jobs_vec(1, job, this->allocator); + this->labelToInsertJobs.insert({label, new_jobs_vec}); + } + return job; + } + + // Buffer a vector in the frontend index as training data for the backend (re)initialization. + // Returns 1 for a new label, 0 for an overwrite, or nullopt if the backend turned ready while + // the guard was taken + std::optional tryBufferForTraining(const void *blob, labelType label, bool in_place) { int ret = 0; - auto svs_index = GetSVSIndex(); - // Backend index deletions to be synchronized with the frontend index, - // elsewhere there is the risk of labels duplication in both indices which can lead to wrong - // results of topK queries. In such case we should behave as if InPlace mode is always set. - bool label_exists = [&]() { - std::shared_lock lock(this->flatIndexGuard); - return this->frontendIndex->isLabelExists(label); - }(); + bool run_init = false; + { + std::lock_guard flat_lock{this->flatIndexGuard}; + // never buffer into a ready backend: ids_to_init_ would have no consumer + if (GetSVSIndex()->ready()) { + return std::nullopt; + } + int deleted = 0; + if (!this->frontendIndex->isMultiValue() && this->frontendIndex->isLabelExists(label)) { + // once the backend has been dropped the buffer may also hold job-backed vectors, + // so the overwrite has to invalidate pending jobs + deleted = removeLabelFromFlat(label); + } + ids_to_init_.insert(this->frontendIndex->indexSize()); + ret = std::max(this->frontendIndex->addVector(blob, label) - deleted, 0); + run_init = ids_to_init_.size() >= this->trainingTriggerThreshold; + if (run_init && !in_place) { + scheduleSVSIndexInit(); + run_init = false; + } + } + if (run_init) { + initSVSIndexWrapper(this, 1); + } + return ret; + } - if (label_exists) { - std::lock_guard lock(this->flatIndexGuard); - ret = this->deleteAndRecordSwaps_Unsafe(label); + // Move a pending job's vector from the insert-job path to the training batch. The vector stays + // in the frontend index, only its ownership changes. + InsertJobOutcome adoptJobIntoInitBuffer(SVSInsertJob *job) { + std::lock_guard flat_lock{this->flatIndexGuard}; + if (!job->isValid) { + std::lock_guard invalid_lock{this->invalidJobsLookupGuard}; + this->invalidJobs.erase(job->id); + return InsertJobOutcome::Completed; + } + if (GetSVSIndex()->ready() || this->indexUpdateScheduled.test(std::memory_order_acquire)) { + return InsertJobOutcome::Deferred; } + ids_to_init_.insert(job->id); + this->detachInsertJob(job); + if (ids_to_init_.size() >= this->trainingTriggerThreshold) { + scheduleSVSIndexInit(); + } + return InsertJobOutcome::Completed; + } - label_exists = [&]() { - std::shared_lock lock(this->mainIndexGuard); - return svs_index->isLabelExists(label); - }(); + int deleteVector(labelType label) override { + int ret = 0; - if (label_exists) { - std::lock_guard lock(this->mainIndexGuard); - ret += this->backendIndex->deleteVector(label); + this->flatIndexGuard.lock_shared(); + if (this->frontendIndex->isLabelExists(label)) { + this->flatIndexGuard.unlock_shared(); + std::lock_guard flat_lock{this->flatIndexGuard}; + ret += removeLabelFromFlat(label); + } else { + this->flatIndexGuard.unlock_shared(); } + + auto deleted = this->backendIndex->deleteVector(label); + if (deleted > 0) { + if (this->getWriteMode() == VecSim_WriteInPlace) { + GetSVSIndex()->consolidate({label}); + } else { + scheduleSVSIndexConsolidate(label); + } + } + ret += deleted; return ret; } + #if HAVE_SVS_REPLACE_EXTERNAL_ID // Only declared when the SVS this was built against offers `replace_external_id`, mirroring // `SVSIndex::relabelVector`. Left out otherwise, so the interface default reports @@ -981,12 +1252,6 @@ class TieredSVSIndex : public VecSimTieredIndex { * Reports `Unsupported` when the backend holds the label and cannot move it, which is the * case when built against an SVS without `replace_external_id`. All-or-nothing: on any code * other than `VecSimRelabel_OK` both tiers are untouched. - * - * `updateJobMutex` is taken first, in the order `updateSVSIndex` takes its own locks. Holding - * it is what makes this correct rather than merely serialised: an update job snapshots the - * buffer's labels *by value* and afterwards reconciles only id swaps and deletions, so a - * rename landing inside its window would be invisible to it and the vector would reach the - * backend under the old label. */ VecSimRelabelCode relabelVector(labelType old_label, labelType new_label) override { if (old_label == new_label) { @@ -994,79 +1259,104 @@ class TieredSVSIndex : public VecSimTieredIndex { } auto *svs_index = GetSVSIndex(); - // Taken together through `std::lock`'s back-off rather than one after another, because - // the orders in this class disagree: nearly everything acquires flat before main, but the - // in-place add path takes `mainIndexGuard` before `flatIndexGuard` while the backend is - // still empty. Acquiring in sequence would hold one guard while blocking on another and - // close a cycle with that path; acquiring them together cannot, whichever order the other - // side uses. Same idiom that path already uses for its own two locks. - std::scoped_lock lock(this->updateJobMutex, this->flatIndexGuard, this->mainIndexGuard); - - const bool in_flat = this->frontendIndex->isLabelExists(old_label); - const bool in_backend = svs_index->isLabelExists(old_label); - if (!in_flat && !in_backend) { - return VecSimRelabel_OldLabelMissing; - } - if (this->frontendIndex->isLabelExists(new_label) || svs_index->isLabelExists(new_label)) { - return VecSimRelabel_NewLabelTaken; - } - - // The backend goes first because it is the tier that can refuse; refusing after the - // buffer had already moved would leave the label half applied. - if (in_backend) { - const VecSimRelabelCode backend_ret = - this->backendIndex->relabelVector(old_label, new_label); - if (backend_ret != VecSimRelabel_OK) { - return backend_ret; - } - } - if (in_flat) { - const VecSimRelabelCode flat_ret = - this->frontendIndex->relabelVector(old_label, new_label); -#ifdef BUILD_TESTS - assert(flat_ret == VecSimRelabel_OK && "the buffer just reported holding this label"); -#endif - UNUSED(flat_ret); - } - return VecSimRelabel_OK; - } + std::shared_lock lock(this->updateJobMutex); -#endif // HAVE_SVS_REPLACE_EXTERNAL_ID + bool flat_holds_old = false; + bool delayed_any = false; + std::vector taken_over; + { + std::lock_guard flat_lock{this->flatIndexGuard}; - size_t getNumMarkedDeleted() const override { - return this->GetSVSIndex()->getNumMarkedDeleted(); - } + flat_holds_old = this->frontendIndex->isLabelExists(old_label); + if (!flat_holds_old && !svs_index->isLabelExists(old_label)) { + return VecSimRelabel_OldLabelMissing; + } + if (this->frontendIndex->isLabelExists(new_label) || + svs_index->isLabelExists(new_label)) { + return VecSimRelabel_NewLabelTaken; + } - size_t indexSize() const override { - std::shared_lock flat_lock(this->flatIndexGuard); - std::shared_lock main_lock(this->mainIndexGuard); - return this->frontendIndex->indexSize() + this->backendIndex->indexSize(); - } + auto pending = this->labelToInsertJobs.find(old_label); + if (pending != this->labelToInsertJobs.end()) { + auto jobs = std::move(pending->second); + this->labelToInsertJobs.erase(pending); + for (auto *job : jobs) { + auto *insert_job = static_cast(job); + // wait it out: a job that published before the remap is remapped with the + // backend, and still drops its own buffer copy under the new label + waitForJob(insert_job); + job->label = new_label; + if (insert_job->status.load(std::memory_order_relaxed) == + SVSInsertJob::Status::Pending) { + // did not publish yet - hold it until the remap settles the label + insert_job->status.store(SVSInsertJob::Status::Delayed, + std::memory_order_relaxed); + delayed_any = true; + } + } + this->labelToInsertJobs.emplace(new_label, std::move(jobs)); + } - size_t indexCapacity() const override { - std::shared_lock flat_lock(this->flatIndexGuard); - std::shared_lock main_lock(this->mainIndexGuard); - return this->frontendIndex->indexCapacity() + this->backendIndex->indexCapacity(); - } + // isLabelExists() said new_label is free, but SVS keeps the translator entry of a + // soft-deleted label until its consolidate job runs, and would refuse the remap. + taken_over = takeOverConsolidateOf(new_label); - double getDistanceFrom_Unsafe(labelType label, const void *blob) const override { - // Try to get the distance from the flat buffer. - // If the label doesn't exist, the distance will be NaN. - auto flat_dist = this->frontendIndex->getDistanceFrom_Unsafe(label, blob); + if (flat_holds_old) { + const VecSimRelabelCode flat_ret = + this->frontendIndex->relabelVector(old_label, new_label); + assert(flat_ret == VecSimRelabel_OK && + "the buffer just reported holding this label"); + UNUSED(flat_ret); + } + } - // Optimization. TODO: consider having different implementations for single and multi - // indexes, to avoid checking the index type on every query. - if (!this->backendIndex->isMultiValue() && !std::isnan(flat_dist)) { - // If the index is single value, and we got a valid distance from the flat buffer, - // we can return the distance without querying the Main index. - return flat_dist; + // Do the taken-over job's, + // the remap below isn't refused by a leftover translator entry + if (!taken_over.empty()) { + svs_index->consolidate(taken_over); + } + + const bool rollback = + this->backendIndex->relabelVector(old_label, new_label) == VecSimRelabel_NewLabelTaken; + const labelType final_label = rollback ? old_label : new_label; + + if (rollback || delayed_any) { + std::lock_guard flat_lock{this->flatIndexGuard}; + if (rollback) { + auto pending = this->labelToInsertJobs.find(new_label); + if (pending != this->labelToInsertJobs.end()) { + auto jobs = std::move(pending->second); + this->labelToInsertJobs.erase(pending); + for (auto *job : jobs) { + job->label = old_label; + } + this->labelToInsertJobs.emplace(old_label, std::move(jobs)); + } + if (flat_holds_old) { + this->frontendIndex->relabelVector(new_label, old_label); + } + } + if (delayed_any) { + auto it = this->labelToInsertJobs.find(final_label); + if (it != this->labelToInsertJobs.end()) { + for (auto *job : it->second) { + auto *insert_job = static_cast(job); + if (insert_job->status.load(std::memory_order_relaxed) == + SVSInsertJob::Status::Delayed) { + insert_job->status.store(SVSInsertJob::Status::Pending, + std::memory_order_relaxed); + } + } + } + } } + return rollback ? VecSimRelabel_NewLabelTaken : VecSimRelabel_OK; + } - // Try to get the distance from the Main index. - auto svs_dist = this->backendIndex->getDistanceFrom_Unsafe(label, blob); +#endif // HAVE_SVS_REPLACE_EXTERNAL_ID - // Return the minimum distance that is not NaN. - return std::fmin(flat_dist, svs_dist); + size_t getNumMarkedDeleted() const override { + return this->GetSVSIndex()->getNumMarkedDeleted(); } VecSimIndexDebugInfo debugInfo() const override { @@ -1077,25 +1367,15 @@ class TieredSVSIndex : public VecSimTieredIndex { .updateTriggerThreshold = this->updateTriggerThreshold, .updateJobWaitTime = this->updateJobWaitTime, }; - { - // Use try_lock to avoid blocking the main thread during long-running - // training operations. updateSVSIndexWrapper holds updateJobMutex for - // the entire training duration (which can take 40-85s on slow machines). - // If the mutex is held, training is actively running, so we report - // indexUpdateScheduled = true (BACKGROUND_INDEXING = 1). - std::unique_lock lock(this->updateJobMutex, std::try_to_lock); - if (lock.owns_lock()) { - svsTieredInfo.indexUpdateScheduled = this->indexUpdateScheduled.test(); - } else { - // Mutex is held by updateSVSIndexWrapper — training is in progress. - svsTieredInfo.indexUpdateScheduled = true; - } - } + + svsTieredInfo.indexUpdateScheduled = + (info.tieredInfo.frontendCommonInfo.indexSize > 0) && + this->indexUpdateScheduled.test(std::memory_order_acquire); info.tieredInfo.specificTieredBackendInfo.svsTieredInfo = svsTieredInfo; + + // Background indexing is in progress whenever the flat buffer is non-empty info.tieredInfo.backgroundIndexing = - svsTieredInfo.indexUpdateScheduled && info.tieredInfo.frontendCommonInfo.indexSize > 0 - ? VecSimBool_TRUE - : VecSimBool_FALSE; + info.tieredInfo.frontendCommonInfo.indexSize > 0 ? VecSimBool_TRUE : VecSimBool_FALSE; return info; } @@ -1151,7 +1431,6 @@ class TieredSVSIndex : public VecSimTieredIndex { TIERED_LOG(VecSimCommonStrings::LOG_VERBOSE_STRING, "running synchronous GC for tiered SVS index in write-in-place mode"); // In write-in-place mode, we run GC synchronously. - std::lock_guard lock{this->mainIndexGuard}; if (this->backendIndex->indexSize() == 0) { // No need to run GC on an empty index. return; @@ -1167,13 +1446,7 @@ class TieredSVSIndex : public VecSimTieredIndex { scheduleSVSIndexGC(); } - void acquireSharedLocks() override { - this->flatIndexGuard.lock_shared(); - this->mainIndexGuard.lock_shared(); - } + void acquireSharedLocks() override { this->flatIndexGuard.lock_shared(); } - void releaseSharedLocks() override { - this->mainIndexGuard.unlock_shared(); - this->flatIndexGuard.unlock_shared(); - } + void releaseSharedLocks() override { this->flatIndexGuard.unlock_shared(); } }; diff --git a/src/VecSim/algorithms/svs/svs_utils.h b/src/VecSim/algorithms/svs/svs_utils.h index 7d90f97c9..232240ae9 100644 --- a/src/VecSim/algorithms/svs/svs_utils.h +++ b/src/VecSim/algorithms/svs/svs_utils.h @@ -16,6 +16,10 @@ #include "svs/core/distance.h" #include "svs/lib/float16.h" #include "svs/index/vamana/dynamic_index.h" +// Pulls in `svs::index::vamana::concurrent` and the `svs::concurrent` alias. The concurrent +// namespace redeclares only the entities it has to replace; every other name (build/search +// parameters, extension points) still resolves to the enclosing `svs::index::vamana`. +#include "svs/concurrent/concurrent.h" #if HAVE_SVS_LVQ #include "svs/cpuid.h" @@ -204,6 +208,21 @@ inline std::pair isSVSQuantBitsSupported(VecSimSvsQuan // unreachable code, but to avoid compiler warning return std::make_pair(VecSimSvsQuant_NONE, false); } + +// The `svs::concurrent` counterpart of `svs::make_blocked_allocator_handle`. +// +// The upstream helper cannot be reused here: it hardcodes `svs::data::Blocked` in its return +// type, so there is no way to ask it for the grow-stable tag. The type-erasing +// `AllocatorHandle` wrapper is what makes the compressed datasets (which take an allocator +// handle rather than a concrete allocator) work with VecSim's own allocator. +template +svs::concurrent::SegmentedBlocked> +make_segmented_blocked_allocator_handle(const svs::data::BlockingParameters ¶meters, + Alloc alloc) { + using handle_type = svs::AllocatorHandle; + return svs::concurrent::SegmentedBlocked{ + parameters, svs::make_allocator_handle(std::move(alloc))}; +} } // namespace svs_details template ; // Used in creating storage + // `SegmentedBlocked` is a drop-in replacement for `svs::data::Blocked` (it derives from + // it and carries the same blocking parameters) that additionally keeps already-published + // elements at a stable address when the dataset grows, which is what allows a concurrent + // reader to keep a pointer into the data while a writer appends. + using blocked_type = svs::concurrent::SegmentedBlocked; // svs::Dynamic means runtime dimensionality in opposite to compile-time dimensionality - using index_storage_type = svs::data::BlockedData; + using index_storage_type = + svs::concurrent::SegmentedBlockedData; static constexpr bool is_compressed() { return false; } @@ -278,9 +302,13 @@ struct SVSStorageTraits { template struct SVSGraphBuilder { using allocator_type = svs_details::SVSAllocator; - using blocked_type = svs::data::Blocked; - using graph_data_type = svs::data::BlockedData; - using graph_type = svs::graphs::SimpleGraph; + using blocked_type = svs::concurrent::SegmentedBlocked; + using graph_data_type = + svs::concurrent::SegmentedBlockedData; + // The concurrent `SimpleGraph` has the same shape as the upstream one but guards each + // adjacency list with a sequence lock, so a reader can traverse an edge list that a + // writer is concurrently rewriting. + using graph_type = svs::concurrent::graphs::SimpleGraph; static blocked_type make_blocked_allocator(size_t block_size, size_t graph_max_degree, std::shared_ptr allocator) { @@ -310,8 +338,8 @@ struct SVSGraphBuilder { // based on the data types, which we found to perform better through heuristic analysis. auto prefetch_parameters = svs::index::vamana::extensions::estimate_prefetch_parameters(data); - auto builder = svs::index::vamana::VamanaBuilder( - graph, data, std::move(distance), parameters, threadpool, prefetch_parameters, logger); + auto builder = svs::concurrent::VamanaBuilder(graph, data, std::move(distance), parameters, + threadpool, prefetch_parameters, logger); // Specific to the Vamana algorithm: // It builds in two rounds, one with alpha=1 and the second time with the user/config @@ -345,6 +373,12 @@ struct SVSGraphBuilder { // graph_max_degree neighbors return sizeof(SVSIdType) * (graph_max_degree + 1); } + + // Per-node size of the concurrent graph's reverse-edge index. + static constexpr size_t reverse_edges_element_size() { + using reverse_edges_type = typename graph_type::reverse_edges_type; + return sizeof(typename reverse_edges_type::list_type) + sizeof(svs::concurrent::SpinLock); + } }; // A slot in the shared SVS thread pool. Wraps an SVS Thread with an occupancy flag diff --git a/src/VecSim/index_factories/svs_factory.cpp b/src/VecSim/index_factories/svs_factory.cpp index 33adf6fb1..bae7c021d 100644 --- a/src/VecSim/index_factories/svs_factory.cpp +++ b/src/VecSim/index_factories/svs_factory.cpp @@ -234,10 +234,12 @@ size_t EstimateElementSize(const SVSParams *params) { // Assuming that the graph_max_degree can be unset in params. const auto graph_max_degree = svs_details::makeVamanaBuildParameters(*params).graph_max_degree; const auto graph_node_size = SVSGraphBuilder::element_size(graph_max_degree); + const auto reverse_edges_node_size = + SVSGraphBuilder::reverse_edges_element_size(); const auto vector_size = QuantizedVectorSize(params->type, params->quantBits, params->dim, 0, params->leanvec_dim); - return vector_size + graph_node_size; + return vector_size + graph_node_size + reverse_edges_node_size; } size_t EstimateInitialSize(const SVSParams *params, bool is_normalized) { diff --git a/src/VecSim/utils/scoped_locks.h b/src/VecSim/utils/scoped_locks.h new file mode 100644 index 000000000..234047a9b --- /dev/null +++ b/src/VecSim/utils/scoped_locks.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2006-Present, Redis Ltd. + * All rights reserved. + * + * Licensed under your choice of the Redis Source Available License 2.0 + * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the + * GNU Affero General Public License v3 (AGPLv3). + */ + +#pragma once + +#include +#include + +// A lockable resource, so that different mutex-like primitives (a shared_mutex, HNSW's own +// data guard, or nothing at all) can be composed under one RAII type below. +class Lockable { +public: + virtual void lock() const = 0; + virtual void unlock() const = 0; + virtual ~Lockable() = default; +}; + +// Adapts a std::shared_mutex, locked/unlocked for shared (read) access, to Lockable. +class SharedMutexLockable : public Lockable { + std::shared_mutex &guard; + +public: + explicit SharedMutexLockable(std::shared_mutex &guard) : guard(guard) {} + void lock() const override { guard.lock_shared(); } + void unlock() const override { guard.unlock_shared(); } +}; + +// RAII acquisition of zero, one or two Lockables, so callers can plug in whatever locking +// (or none) they need without repeating the lock/unlock bookkeeping. +class ScopedLocks { + std::array locks{}; + size_t count = 0; + +public: + ScopedLocks() = default; + explicit ScopedLocks(const Lockable &a) : locks{&a, nullptr}, count(1) { a.lock(); } + ScopedLocks(const Lockable &a, const Lockable &b) : locks{&a, &b}, count(2) { + a.lock(); + b.lock(); + } + ScopedLocks(ScopedLocks &&other) noexcept : locks(other.locks), count(other.count) { + other.count = 0; + } + ScopedLocks(const ScopedLocks &) = delete; + ScopedLocks &operator=(ScopedLocks &&) = delete; + ~ScopedLocks() { + // Unlock in reverse acquisition order. + for (size_t i = count; i-- > 0;) { + locks[i]->unlock(); + } + } +}; diff --git a/src/VecSim/vec_sim_common.h b/src/VecSim/vec_sim_common.h index 5f465c071..6f0843236 100644 --- a/src/VecSim/vec_sim_common.h +++ b/src/VecSim/vec_sim_common.h @@ -304,7 +304,9 @@ typedef enum { HNSW_SWAP_JOB, HNSW_DISK_JOB, SVS_BATCH_UPDATE_JOB, + SVS_INSERT_VECTOR_JOB, SVS_GC_JOB, + SVS_CONSOLIDATE_JOB, INVALID_JOB // to indicate that finding a JobType >= INVALID_JOB is an error } JobType; diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 286d4f057..1e99e765b 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -15,8 +15,10 @@ #include "VecSim/tombstone_interface.h" #include "VecSim/utils/query_result_utils.h" #include "VecSim/utils/alignment.h" +#include "VecSim/utils/scoped_locks.h" #include +#include #include #include @@ -42,6 +44,19 @@ struct AsyncJob : public VecsimBaseObject { isValid(true) {} }; +/** + * Definition of a job that inserts a new vector from flat into the backend index. + * Backend specific insert jobs derive from it to set their own job type. + */ +struct TieredInsertJob : public AsyncJob { + labelType label; + idType id; + + TieredInsertJob(std::shared_ptr allocator, JobType type, labelType label_, + idType id_, JobCallback insertCb, VecSimIndex *index_) + : AsyncJob(allocator, type, insertCb, index_), label(label_), id(id_) {} +}; + // All read operations (including KNN, range, batch iterators and get-distance-from) are guaranteed // to consider all vectors that were added to the index before the query was submitted. The results // may include vectors that were added after the query was submitted, with no guarantees. @@ -57,6 +72,18 @@ class VecSimTieredIndex : public VecSimIndexInterface { mutable std::shared_mutex flatIndexGuard; mutable std::shared_mutex mainIndexGuard; + SharedMutexLockable flatIndexLockable{flatIndexGuard}; + SharedMutexLockable mainIndexLockable{mainIndexGuard}; + + // Locking behavior for topKQuery/rangeQuery + virtual ScopedLocks lockMainIndexForQuery() const = 0; + + // Locking behavior for indexSize() + virtual ScopedLocks lockIndexForSize() const = 0; + + // Locking behavior for indexCapacity() + virtual ScopedLocks lockIndexForCapacity() const = 0; + void lockMainIndexGuard() const { mainIndexGuard.lock(); #ifdef BUILD_TESTS @@ -121,6 +148,91 @@ class VecSimTieredIndex : public VecSimIndexInterface { VecSimQueryParams *queryParams, VecSimQueryReply_Order order) const; +#ifdef BUILD_TESTS +public: +#endif + /// Mappings from id/label to associated jobs, for invalidating and update ids if necessary. + // In MULTI, we can have more than one insert job pending per label. + // **This map is protected with the flat buffer lock** + vecsim_stl::unordered_map> labelToInsertJobs; + + // Helper function for updating the pending insert job(s) of a label after the flat buffer + // swapped the vector's internal id + virtual void updateInsertJobInternalId(idType prev_id, idType new_id, labelType label) { + // Update the pending job id, due to a swap that was caused after the removal of new_id. + assert(new_id != INVALID_ID && prev_id != INVALID_ID); + auto it = this->labelToInsertJobs.find(label); + if (it != this->labelToInsertJobs.end()) { + // There is a pending job for the label of the swapped last id - update its id. + for (TieredInsertJob *job_it : it->second) { + if (job_it->id == prev_id) { + job_it->id = new_id; + } + } + } + } + + // A mapping to hold invalid jobs, so we can dispose them upon index deletion. + vecsim_stl::unordered_map invalidJobs; + idType currInvalidJobId; // A unique arbitrary identifier for accessing invalid jobs + std::mutex invalidJobsLookupGuard; + + // Set an insert/repair job as invalid, put the job pointer in the invalid jobs lookup under + // the current available id, increase it and return it (while holding invalidJobsLookupGuard). + // Returns the id that the job was stored under (to be set in the job id field). + virtual idType setAndSaveInvalidJob(AsyncJob *job) { + std::lock_guard lock(this->invalidJobsLookupGuard); + job->isValid = false; + idType curInvalidId = currInvalidJobId++; + this->invalidJobs.insert({curInvalidId, job}); + return curInvalidId; + } + + // Remove the job pointer from the labelToInsertJobs mapping. Must hold flatIndexGuard. + void detachInsertJob(TieredInsertJob *job) { + auto it = this->labelToInsertJobs.find(job->label); + if (it == this->labelToInsertJobs.end()) { + return; + } + auto &jobs = it->second; + for (size_t i = 0; i < jobs.size(); i++) { + if (jobs[i] == job) { + jobs.erase(jobs.begin() + (long)i); + break; + } + } + if (jobs.empty()) { + this->labelToInsertJobs.erase(it); + } + } + + // Remove a vector and its insert job from the flat buffer + void removeIngestedVectorFromFlat(TieredInsertJob *job) { + std::lock_guard flat_lock(this->flatIndexGuard); + // The job might have been invalidated due to overwrite in the meantime. In this case, + // it was already deleted and the job has been evicted. Otherwise, we need to do it now. + if (!job->isValid) { + // Remove the current job from the invalid jobs' lookup, as we are about to delete it + // now. + std::lock_guard invalid_jobs_lock(this->invalidJobsLookupGuard); + this->invalidJobs.erase(job->id); + return; + } + this->detachInsertJob(job); + // Remove the vector from the flat buffer. This may cause the last vector id to swap with + // the deleted id. Hold the label for the last id, so we can later on update its + // corresponding job id. Note that after calling deleteVectorById, the last id's label + // shouldn't be available, since it is removed from the lookup. + labelType last_vec_label = + this->frontendIndex->getVectorLabel(this->frontendIndex->indexSize() - 1); + int deleted = this->frontendIndex->deleteVectorById(job->label, job->id); + if (deleted && job->id != this->frontendIndex->indexSize()) { + // If the vector removal caused a swap with the last id, update the relevant insert job. + this->updateInsertJobInternalId(this->frontendIndex->indexSize(), job->id, + last_vec_label); + } + } + public: /** * @brief Get the vector elements stored under a label, in insertion order. @@ -188,17 +300,72 @@ class VecSimTieredIndex : public VecSimIndexInterface { } } + // `getDistanceFrom` returns the minimum distance between the given blob and the vector with + // the given label. If the label doesn't exist, the distance will be NaN. + // Therefore, it's better to just call `getDistanceFrom` on both indexes and return the minimum + // instead of checking if the label exists in each index. We first try to get the distance from + // the flat buffer, as vectors in the buffer might move to the backend while we're "between" + // the locks. + // Behavior for single (regular) index: + // 1. label doesn't exist in both indexes - return NaN + // 2. label exists in one of the indexes only - return the distance from that index (valid) + // 3. label exists in both indexes - return the value from the flat buffer (valid and equal to + // the value from the backend index), saving us from locking the backend index. + // Behavior for multi index: + // 1. label doesn't exist in both indexes - return NaN + // 2. label exists in one of the indexes only - return the distance from that index (valid) + // 3. label exists in both indexes - we may have some of the vectors with the same label in the + // flat buffer only and some in the backend index only (and maybe temporal duplications). So, + // we get the distance from both indexes and return the minimum. + // + // IMPORTANT: this should be called when the *tiered index locks are locked for shared + // ownership*, along with the backend index's own data guard lock if it has one. That is since + // the internal getDistanceFrom calls access the indexes' data, and it is not safe to run + // insert/delete operations in parallel. Also, we avoid acquiring the locks internally, since + // this is usually called for every vector individually, and the overhead of acquiring and + // releasing the locks is significant in that case. + double getDistanceFrom_Unsafe(labelType label, const void *blob) const override { + // Try to get the distance from the flat buffer. + // If the label doesn't exist, the distance will be NaN. + auto flat_dist = this->frontendIndex->getDistanceFrom_Unsafe(label, blob); + + // Optimization. TODO: consider having different implementations for single and multi + // indexes, to avoid checking the index type on every query. + if (!this->backendIndex->isMultiValue() && !std::isnan(flat_dist)) { + // If the index is single value, and we got a valid distance from the flat buffer, + // we can return the distance without querying the backend index. + return flat_dist; + } + + // Try to get the distance from the backend index. + auto backend_dist = this->backendIndex->getDistanceFrom_Unsafe(label, blob); + + // Return the minimum distance that is not NaN. + return std::fmin(flat_dist, backend_dist); + } + VecSimTieredIndex(VecSimIndexAbstract *backendIndex_, BruteForceIndex *frontendIndex_, TieredIndexParams tieredParams, std::shared_ptr allocator) : VecSimIndexInterface(allocator), backendIndex(backendIndex_), frontendIndex(frontendIndex_), jobQueue(tieredParams.jobQueue), jobQueueCtx(tieredParams.jobQueueCtx), SubmitJobsToQueue(tieredParams.submitCb), - flatBufferLimit(tieredParams.flatBufferLimit) { + flatBufferLimit(tieredParams.flatBufferLimit), labelToInsertJobs(this->allocator), + invalidJobs(this->allocator), currInvalidJobId(0) { assert(backendIndex != nullptr); } virtual ~VecSimTieredIndex() { + // Delete all the pending insert jobs. + for (auto &jobs : this->labelToInsertJobs) { + for (auto *job : jobs.second) { + delete job; + } + } + // Delete all the pending invalid jobs. + for (auto &it : this->invalidJobs) { + delete it.second; + } VecSimIndex_Free(backendIndex); VecSimIndex_Free(frontendIndex); } @@ -210,6 +377,16 @@ class VecSimTieredIndex : public VecSimIndexInterface { VecSimQueryParams *queryParams, VecSimQueryReply_Order order) const override; + size_t indexSize() const override { + auto locks = lockIndexForSize(); + return this->frontendIndex->indexSize() + this->backendIndex->indexSize(); + } + + size_t indexCapacity() const override { + auto locks = lockIndexForCapacity(); + return this->frontendIndex->indexCapacity() + this->backendIndex->indexCapacity(); + } + virtual inline uint64_t getAllocationSize() const override { return this->allocator->getAllocationSize() + this->backendIndex->getAllocationSize() + this->frontendIndex->getAllocationSize(); @@ -255,8 +432,10 @@ VecSimTieredIndex::topKQueryImp(const void *queryBlob, size_ // Simply query the main index and return the results while holding the lock. auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); - std::shared_lock main_lock(this->mainIndexGuard); - return this->backendIndex->topKQuery(processed_query, k, queryParams); + auto mainLock = lockMainIndexForQuery(); + auto res = this->backendIndex->topKQuery(processed_query, k, queryParams); + + return res; } else { // No luck... first query the flat buffer and release the lock. // The query blob is already processed according to the frontend index. @@ -271,10 +450,12 @@ VecSimTieredIndex::topKQueryImp(const void *queryBlob, size_ auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); - // Lock the main index and query it. - std::shared_lock main_lock(this->mainIndexGuard); - auto main_results = this->backendIndex->topKQuery(processed_query, k, queryParams); - main_lock.unlock(); + VecSimQueryReply *main_results; + { + // Lock the main index and query it. + auto mainLock = lockMainIndexForQuery(); + main_results = this->backendIndex->topKQuery(processed_query, k, queryParams); + } // If the query failed (currently only on timeout), return the error code. if (main_results->code != VecSim_QueryReply_OK) { @@ -317,10 +498,12 @@ VecSimTieredIndex::rangeQueryImp(const void *queryBlob, doub auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); - // Simply query the main index and return the results while holding the lock. - this->mainIndexGuard.lock_shared(); - auto res = this->backendIndex->rangeQuery(processed_query, radius, queryParams); - this->mainIndexGuard.unlock_shared(); + VecSimQueryReply *res; + { + auto mainLock = lockMainIndexForQuery(); + // Simply query the main index and return the results while holding the lock. + res = this->backendIndex->rangeQuery(processed_query, radius, queryParams); + } // We could have passed the order to the main index, but we can sort them here after // unlocking it instead. @@ -341,9 +524,12 @@ VecSimTieredIndex::rangeQueryImp(const void *queryBlob, doub auto processed_query_ptr = this->frontendIndex->preprocessQuery(queryBlob); const void *processed_query = processed_query_ptr.get(); // Lock the main index and query it. - this->mainIndexGuard.lock_shared(); - auto main_results = this->backendIndex->rangeQuery(processed_query, radius, queryParams); - this->mainIndexGuard.unlock_shared(); + + VecSimQueryReply *main_results; + { + auto mainLock = lockMainIndexForQuery(); + main_results = this->backendIndex->rangeQuery(processed_query, radius, queryParams); + } // Merge the results and return, avoiding duplicates. // At this point, the return code of the FLAT index is OK, and the return code of the MAIN diff --git a/src/python_bindings/bindings.cpp b/src/python_bindings/bindings.cpp index d1e3b0792..9ac584c5f 100644 --- a/src/python_bindings/bindings.cpp +++ b/src/python_bindings/bindings.cpp @@ -213,7 +213,9 @@ class PyVecSimIndex { return PyBatchIterator(index, py_batch_ptr); } - void runGC() { VecSimTieredIndex_GC(index.get()); } + // Not VecSimTieredIndex_GC(): that is a no-op unless the index is tiered, while + // SVSIndex::runGC() compacts and rebuilds reverse edges on a plain index too. + void runGC() { index->runGC(); } VecSimRelabelCode relabelVector(labelType old_label, labelType new_label) { py::gil_scoped_release py_gil; diff --git a/tests/flow/test_svs_tiered.py b/tests/flow/test_svs_tiered.py index 09ab8c93b..876aad8f8 100644 --- a/tests/flow/test_svs_tiered.py +++ b/tests/flow/test_svs_tiered.py @@ -166,6 +166,9 @@ def create_tiered_index(test_logger, is_multi: bool, num_per_label=1, data_type= test_logger.info(f"Total time for inserting vectors to the tiered index and indexing them into SVS using {threads_num}" f" threads took {round_ms(tiered_index_time)} ms") + index.run_gc() + index.wait_for_index() + # Measure total memory of the tiered index. tiered_memory = bytes_to_mega(index.index_memory()) @@ -175,6 +178,8 @@ def create_tiered_index(test_logger, is_multi: bool, num_per_label=1, data_type= _, svs_index_time, _ = indices_ctx.populate_index(svs_index) test_logger.info(f"Insert {num_elements} vectors directly to SVS index (one by one) took {round_(svs_index_time)} s") + + svs_index.run_gc() svs_memory = bytes_to_mega(svs_index.index_memory()) test_logger.info(f"total memory of svs index = {svs_memory} MB") diff --git a/tests/unit/test_hnsw_tiered.cpp b/tests/unit/test_hnsw_tiered.cpp index 9b3600cdf..46afdede6 100644 --- a/tests/unit/test_hnsw_tiered.cpp +++ b/tests/unit/test_hnsw_tiered.cpp @@ -124,7 +124,7 @@ TYPED_TEST(HNSWTieredIndexTest, CreateIndexInstance) { auto job = new (allocator) HNSWInsertJob(tiered_index->allocator, vector_label, 0, insert_to_index, tiered_index); - auto jobs_vec = vecsim_stl::vector(1, job, allocator); + auto jobs_vec = vecsim_stl::vector(1, job, allocator); tiered_index->labelToInsertJobs.insert({vector_label, jobs_vec}); // Wrap this job with an array and submit the jobs to the queue. @@ -339,10 +339,10 @@ TYPED_TEST(HNSWTieredIndexTest, addVector) { sizeof(void *) + sizeof(size_t); // Account for the memory that was allocated in the labelToInsertJobs map (approx.) expected_mem += - sizeof( - vecsim_stl::unordered_map>::value_type) + + sizeof(vecsim_stl::unordered_map>::value_type) + sizeof(void *) + sizeof(size_t); - // Account for the inner buffer of the std::vector in the map. + // Account for the inner buffer of the std::vector in the map. expected_mem += sizeof(void *) + sizeof(size_t); // Account for the insert job that was created. expected_mem += sizeof(HNSWInsertJob) + sizeof(size_t); @@ -4988,7 +4988,7 @@ TYPED_TEST(HNSWTieredIndexTestBasic, relabelVectorMulti) { } ASSERT_EQ(tiered_index->labelToInsertJobs.count(7), 0); ASSERT_EQ(tiered_index->labelToInsertJobs.at(70).size(), per_label); - for (HNSWInsertJob *job : tiered_index->labelToInsertJobs.at(70)) { + for (TieredInsertJob *job : tiered_index->labelToInsertJobs.at(70)) { ASSERT_EQ(job->label, 70); } diff --git a/tests/unit/test_svs.cpp b/tests/unit/test_svs.cpp index 59ee2798b..164da83e7 100644 --- a/tests/unit/test_svs.cpp +++ b/tests/unit/test_svs.cpp @@ -35,6 +35,16 @@ } \ } +// Expected allocation for one block of `block_size` element slots, of which only +// `num_elements` are actually populated. +// +static size_t EstimateBlockSize(const SVSParams ¶ms, size_t block_size, size_t num_elements) { + const size_t reverse_edges_per_slot = SVSGraphBuilder::reverse_edges_element_size(); + const size_t reverse_edges_slots = svs::lib::SegmentedVector(num_elements).capacity(); + return (EstimateElementSize(params) - reverse_edges_per_slot) * block_size + + reverse_edges_per_slot * reverse_edges_slots; +} + // Log callback function to print non-debug log messages static void svsTestLogCallBackNoDebug(void *ctx, const char *level, const char *message) { if (level == nullptr || message == nullptr) { @@ -1996,7 +2006,7 @@ TYPED_TEST(SVSTest, testSizeEstimation) { size_t actual = index->getAllocationSize(); ASSERT_EQ(estimation, actual); - estimation = EstimateElementSize(params) * bs; + estimation = EstimateBlockSize(params, bs, 1); GenerateAndAddVector(index, dim, 0); actual = index->getAllocationSize() - actual; // get the delta @@ -2909,7 +2919,7 @@ TEST(SVSTest, quant_modes) { ASSERT_EQ(VecSimIndex_IndexSize(index), n); - estimation = EstimateElementSize(params) * params.blockSize; + estimation = EstimateBlockSize(params, params.blockSize, n); actual = index->getAllocationSize() - actual; // get the delta ASSERT_GT(actual, 0); // LVQ element size estimation accuracy is low @@ -3271,7 +3281,7 @@ TEST(SVSTest, scalar_quantization_query) { ASSERT_EQ(VecSimIndex_IndexSize(index_sq), n); ASSERT_EQ(index_sq->indexCapacity(), n); - estimation = EstimateElementSize(params) * params.blockSize; + estimation = EstimateBlockSize(params, params.blockSize, n); actual = index_sq->getAllocationSize() - actual; // get the delta ASSERT_GT(actual, 0); ASSERT_GE(estimation * 1.01, actual); diff --git a/tests/unit/test_svs_fp16.cpp b/tests/unit/test_svs_fp16.cpp index fcf2c7dd7..63ccf1aab 100644 --- a/tests/unit/test_svs_fp16.cpp +++ b/tests/unit/test_svs_fp16.cpp @@ -40,6 +40,13 @@ using float16 = vecsim_types::float16; } \ } +static size_t EstimateBlockSize(const SVSParams ¶ms, size_t block_size, size_t num_elements) { + const size_t reverse_edges_per_slot = SVSGraphBuilder::reverse_edges_element_size(); + const size_t reverse_edges_slots = svs::lib::SegmentedVector(num_elements).capacity(); + return (EstimateElementSize(params) - reverse_edges_per_slot) * block_size + + reverse_edges_per_slot * reverse_edges_slots; +} + // Log callback function to print non-debug log messages static void svsTestLogCallBackNoDebug(void *ctx, const char *level, const char *message) { if (level == nullptr || message == nullptr) { @@ -1388,7 +1395,7 @@ TYPED_TEST(FP16SVSTest, testSizeEstimation) { size_t actual = index->getAllocationSize(); ASSERT_EQ(estimation, actual); - estimation = EstimateElementSize(params) * bs; + estimation = EstimateBlockSize(params, bs, 1); this->GenerateAndAddVector(index, dim, 0); actual = index->getAllocationSize() - actual; // get the delta @@ -1589,7 +1596,7 @@ TYPED_TEST(FP16SVSTest, quant_modes) { ASSERT_EQ(VecSimIndex_IndexSize(index), n); - estimation = EstimateElementSize(params) * DEFAULT_BLOCK_SIZE; + estimation = EstimateBlockSize(params, DEFAULT_BLOCK_SIZE, n); actual = index->getAllocationSize() - actual; // get the delta ASSERT_GT(actual, 0); // LVQ element size estimation accuracy is low @@ -1666,15 +1673,19 @@ TYPED_TEST(FP16SVSTest, scalar_quantization_query) { const size_t k = 10; const double quant_precision = 1.0 / (1 << 7); // int8 quantization precision + std::default_random_engine gen; + std::uniform_real_distribution dist(-1.0, 1.0); std::vector> dataset(n); for (size_t i = 0; i < n; i++) { - test_utils::populate_float16_vec(dataset[i].data(), dim, i, -1.0, 1.0); + for (size_t j = 0; j < dim; j++) { + dataset[i][j] = vecsim_types::FP32_to_FP16(dist(gen)); + } } std::vector ids(n); std::iota(ids.begin(), ids.end(), 0); float16 query[dim]; - test_utils::populate_float16_vec(query, dim, n, -1.0, 1.0); + this->GenerateVector(query, dim, 0.1f); VecSimQueryReply *fp_results = nullptr; auto verify_res = [&](size_t id, double score, size_t result_rank) { @@ -1724,7 +1735,7 @@ TYPED_TEST(FP16SVSTest, scalar_quantization_query) { ASSERT_EQ(VecSimIndex_IndexSize(index_sq), n); ASSERT_EQ(index_sq->indexCapacity(), n); - estimation = EstimateElementSize(params) * params.blockSize; + estimation = EstimateBlockSize(params, params.blockSize, n); actual = index_sq->getAllocationSize() - actual; // get the delta ASSERT_GT(actual, 0); ASSERT_GE(estimation * 1.01, actual); @@ -2328,7 +2339,7 @@ class FP16SVSTieredIndexTest : public FP16SVSTest { } } // Submit the index update job. - tiered_index->scheduleSVSIndexUpdate(); + tiered_index->scheduleSVSIndexInit(); ASSERT_EQ(mock_thread_pool.jobQ.size(), mock_thread_pool.thread_pool_size); // Execute the job from the queue and validate that the index was updated properly. @@ -2848,8 +2859,9 @@ TYPED_TEST(FP16SVSTieredIndexTest, deleteVector) { ASSERT_EQ(tiered_index->indexSize(), 1); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 1); - // Move the vector to SVS by executing the insert job. - mock_thread_pool.thread_iteration(); + // Move the vector to SVS by executing the pending jobs. + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->indexLabelCount(), 1); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 1); // Scalar quantization accuracy is insufficient for this check. @@ -2896,7 +2908,9 @@ TYPED_TEST(FP16SVSTieredIndexTest, deleteVectorMulti) { ASSERT_EQ(tiered_index->deleteVector(vec_label), 2); ASSERT_EQ(tiered_index->indexSize(), 0); ASSERT_EQ(tiered_index->indexLabelCount(), 0); - mock_thread_pool.thread_iteration(); + + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); // Test deleting a label for which both of its vector's is in the flat index. @@ -2910,13 +2924,21 @@ TYPED_TEST(FP16SVSTieredIndexTest, deleteVectorMulti) { ASSERT_EQ(tiered_index->deleteVector(vec_label), 2); ASSERT_EQ(tiered_index->indexLabelCount(), 0); + + ASSERT_EQ(mock_thread_pool.jobQ.size(), 2); + mock_thread_pool.thread_iteration(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); mock_thread_pool.thread_iteration(); ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); // Test deleting a label for which both of its vector's is in SVS index. this->GenerateAndAddVector(tiered_index, dim, vec_label, vec_label); this->GenerateAndAddVector(tiered_index, dim, vec_label, other_vec_val); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 2); + mock_thread_pool.thread_iteration(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); mock_thread_pool.thread_iteration(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); ASSERT_EQ(tiered_index->indexLabelCount(), 1); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 2); diff --git a/tests/unit/test_svs_multi.cpp b/tests/unit/test_svs_multi.cpp index eb87379d9..5e43b4b02 100644 --- a/tests/unit/test_svs_multi.cpp +++ b/tests/unit/test_svs_multi.cpp @@ -23,6 +23,13 @@ } \ } +static size_t EstimateBlockSize(const SVSParams ¶ms, size_t block_size, size_t num_elements) { + const size_t reverse_edges_per_slot = SVSGraphBuilder::reverse_edges_element_size(); + const size_t reverse_edges_slots = svs::lib::SegmentedVector(num_elements).capacity(); + return (EstimateElementSize(params) - reverse_edges_per_slot) * block_size + + reverse_edges_per_slot * reverse_edges_slots; +} + // Log callback function to print non-debug log messages static void svsTestLogCallBackNoDebug(void *ctx, const char *level, const char *message) { if (level == nullptr || message == nullptr) { @@ -748,7 +755,7 @@ TYPED_TEST(SVSMultiTest, testSizeEstimation) { size_t actual = index->getAllocationSize(); ASSERT_EQ(estimation, actual); - estimation = EstimateElementSize(params) * bs; + estimation = EstimateBlockSize(params, bs, 1); GenerateAndAddVector(index, dim, 0); actual = index->getAllocationSize() - actual; // get the delta diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 5fa8a7e3d..005411a67 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -29,6 +29,11 @@ } \ } +static size_t EstimateReverseEdgesSize(size_t num_elements) { + return SVSGraphBuilder::reverse_edges_element_size() * + svs::lib::SegmentedVector(num_elements).capacity(); +} + // Get available number of CPUs // Returns the number of logical processors on the process // Returns std::thread::hardware_concurrency() if the number of logical processors is not available @@ -713,7 +718,7 @@ TYPED_TEST(SVSTieredIndexTest, TestDebugInfoThreadCountWriteInPlace) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); } // Verify: numThreads (pool size) unchanged, lastReservedThreads (parallelism) reflects - // we only used one thread (write-in-place calls updateSVSIndexWrapper with availableThreads=1). + // we only used one thread (write-in-place calls initSVSIndexWrapper with availableThreads=1). backendIndexInfo = tiered_index->GetBackendIndex()->debugInfo(); ASSERT_EQ(backendIndexInfo.svsInfo.numThreads, num_threads); ASSERT_EQ(backendIndexInfo.svsInfo.lastReservedThreads, 1); @@ -798,7 +803,7 @@ TYPED_TEST(SVSTieredIndexTest, CreateIndexInstance) { ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 0); // Submit the index update job. - tiered_index->scheduleSVSIndexUpdate(); + tiered_index->scheduleSVSIndexInit(); ASSERT_EQ(mock_thread_pool.jobQ.size(), mock_thread_pool.thread_pool_size); // Execute the job from the queue and validate that the index was updated properly. @@ -855,7 +860,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, getDataByLabelReadsSvsBackendWhenUncompresse ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 1); // Ingest it into the SVS backend. - tiered_index->scheduleSVSIndexUpdate(); + tiered_index->scheduleSVSIndexInit(); mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 1); @@ -937,7 +942,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, ShrinkDuringScheduledUpdateIsDeferred) { shrink_callback_ran = true; }); - tiered_index->scheduleSVSIndexUpdate(); + tiered_index->scheduleSVSIndexInit(); ASSERT_EQ(mock_thread_pool.jobQ.size(), num_threads); mock_thread_pool.init_threads(); @@ -1021,7 +1026,9 @@ TYPED_TEST(SVSTieredIndexTestBasic, ShrinkDuringScheduledGCIsDeferred) { }); VecSimTieredIndex_GC(tiered_index); - ASSERT_EQ(mock_thread_pool.jobQ.size(), num_threads); + // Each soft-deleted slot has corresponding consolidate() job + size_t num_consolidate = tiered_index->GetSVSIndex()->getNumMarkedDeleted(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), num_threads + num_consolidate); mock_thread_pool.init_threads(); mock_thread_pool.thread_pool_join(); @@ -1089,14 +1096,16 @@ TYPED_TEST(SVSTieredIndexTest, addVector) { ASSERT_LE(expected_mem, tiered_index->getAllocationSize()); if constexpr (TypeParam::isMulti()) { - // Add another vector under the same label + // Add another vector under the same label. The backend init was already submitted + // by the first vector (thread_pool_size batch-init jobs), so this second vector takes + // the per-vector async path and schedules one additional SVSInsertJob of its own. VecSimIndex_AddVector(tiered_index, vector, vec_label); ASSERT_EQ(tiered_index->indexSize(), 2); ASSERT_EQ(tiered_index->indexLabelCount(), 1); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 2); - // Validate that there still 1 update jobs set - ASSERT_EQ(mock_thread_pool.jobQ.size(), mock_thread_pool.thread_pool_size); + // The batch-init jobs plus the extra per-vector insert job. + ASSERT_EQ(mock_thread_pool.jobQ.size(), mock_thread_pool.thread_pool_size + 1); } } @@ -1146,6 +1155,12 @@ TYPED_TEST(SVSTieredIndexTest, background_indexing_check) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); } + // The backgroundIndexing flag tracks the batch update job only; individual async + // SVS_INSERT_VECTOR_JOBs may still be in flight, each transiently holding its vector in + // both the flat and backend indices. Drain the queue so the total size is stable before + // asserting the exact count. + mock_thread_pool.thread_pool_join(); + ASSERT_GT(tiered_index->GetBackendIndex()->indexSize(), training_th + second_batch / update_th); ASSERT_LT(tiered_index->GetFlatIndex()->indexSize(), update_th); ASSERT_EQ(tiered_index->indexSize(), second_batch + training_th); @@ -1612,8 +1627,9 @@ TYPED_TEST(SVSTieredIndexTest, deleteVector) { ASSERT_EQ(tiered_index->indexSize(), 1); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 1); - // Move the vector to SVS by executing the insert job. - mock_thread_pool.thread_iteration(); + // Move the vector to SVS by executing the pending jobs. + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->indexLabelCount(), 1); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 1); // Scalar quantization accuracy is insufficient for this check. @@ -1654,7 +1670,8 @@ TYPED_TEST(SVSTieredIndexTestBasic, markedDeleted) { ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 0); // Move vectors to the backend - mock_thread_pool.thread_iteration(); + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), n); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->indexSize(), n); @@ -1736,6 +1753,8 @@ TYPED_TEST(SVSTieredIndexTestBasic, deleteVectorMulti) { ASSERT_EQ(tiered_index->deleteVector(vec_label), 2); ASSERT_EQ(tiered_index->indexSize(), 0); ASSERT_EQ(tiered_index->indexLabelCount(), 0); + + mock_thread_pool.thread_iteration(); mock_thread_pool.thread_iteration(); ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); @@ -1748,13 +1767,20 @@ TYPED_TEST(SVSTieredIndexTestBasic, deleteVectorMulti) { ASSERT_EQ(tiered_index->indexSize(), 2); ASSERT_EQ(tiered_index->deleteVector(vec_label), 2); ASSERT_EQ(tiered_index->indexLabelCount(), 0); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 2); + mock_thread_pool.thread_iteration(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); mock_thread_pool.thread_iteration(); ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); // Test deleting a label for which both of its vector's is in SVS index. GenerateAndAddVector(tiered_index, dim, vec_label, vec_label); GenerateAndAddVector(tiered_index, dim, vec_label, other_vec_val); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 2); + mock_thread_pool.thread_iteration(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 1); mock_thread_pool.thread_iteration(); + ASSERT_EQ(mock_thread_pool.jobQ.size(), 0); ASSERT_EQ(tiered_index->indexLabelCount(), 1); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 2); @@ -2074,8 +2100,9 @@ TYPED_TEST(SVSTieredIndexTest, testSizeEstimation) { } mock_thread_pool.thread_pool_wait(); - // Estimate memory delta for filling up the first block and adding another block. - size_t estimation = VecSimIndex_EstimateElementSize(¶ms) * bs; + const size_t reverse_edges_per_slot = SVSGraphBuilder::reverse_edges_element_size(); + size_t estimation = (VecSimIndex_EstimateElementSize(¶ms) - reverse_edges_per_slot) * bs + + EstimateReverseEdgesSize(n + 1) - EstimateReverseEdgesSize(n); size_t before = index->getAllocationSize(); GenerateAndAddVector(index, dim, bs + n, bs + n); @@ -2148,7 +2175,7 @@ TYPED_TEST(SVSTieredIndexTest, parallelInsertAdHoc) { tiered_index->submitSingleJob(search_job); } - tiered_index->scheduleSVSIndexUpdate(); + tiered_index->scheduleSVSIndexInit(); mock_thread_pool.thread_pool_join(); EXPECT_EQ(successful_searches, n); @@ -3170,7 +3197,8 @@ TYPED_TEST(SVSTieredIndexTestBasic, overwriteVectorBasic) { ASSERT_EQ(tiered_index->getDistanceFrom_Unsafe(0, overwritten_vec), 0); // Ingest the updated vector to SVS. - mock_thread_pool.thread_iteration(); + while (mock_thread_pool.jobQ.size() > 0) + mock_thread_pool.thread_iteration(); ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 1); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); ASSERT_EQ(tiered_index->indexLabelCount(), 1); @@ -3449,8 +3477,10 @@ TYPED_TEST(SVSTieredIndexTest, writeInPlaceMode) { ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), 2); ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); - // Overwrite inplace - only in single-value mode + // In write-in-place mode every mutation runs synchronously, including consolidation + // of soft-deleted slots. size_t expected_marked_deleted = 0; + // Overwrite inplace - only in single-value mode if (!TypeParam::isMulti()) { TEST_DATA_T overwritten_vec[] = {1, 1, 1, 1}; tiered_index->addVector(overwritten_vec, vec_label); @@ -3462,7 +3492,8 @@ TYPED_TEST(SVSTieredIndexTest, writeInPlaceMode) { ASSERT_EQ(tiered_index->getDistanceFrom_Unsafe(vec_label, overwritten_vec), 0); ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), expected_marked_deleted); } - // Validate that the vector is marked as deleted. + // Validate that the vector is deleted and consolidated synchronously (in-place mode), + // so no marked-deleted entry remains. tiered_index->deleteVector(vec_label); expected_marked_deleted++; ASSERT_EQ(tiered_index->GetBackendIndex()->indexSize(), TypeParam::isMulti() ? 2 : 3); @@ -3660,11 +3691,12 @@ TYPED_TEST(SVSTieredIndexTestBasic, runGCAPI) { auto jobs_before_gc = mock_thread_pool.jobQ.size(); // Run the GC API call, expect that we will clean up the SVS index. VecSimTieredIndex_GC(tiered_index); - // Expected that GC jobs were added to the queue. + // Expected that a multi-thread GC job was added to the queue. ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + mock_thread_pool.thread_pool_size); // Run GC twice. VecSimTieredIndex_GC(tiered_index); - // Expected that no new GC jobs were added to the queue. + // Expected that no new GC jobs were added to the queue (indexGCScheduled is still set until the + // pending jobs run). ASSERT_EQ(mock_thread_pool.jobQ.size(), jobs_before_gc + mock_thread_pool.thread_pool_size); // Wait for any pending jobs to complete. As far as SVS GC is done via a job. mock_thread_pool.init_threads(); @@ -3933,7 +3965,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalSingle) { // update job paused, we have vectors 0-(n-1) in the index, let's do index modifications // Remove vector label=n-2, it is copied to backend index. - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 2), 2); + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 2), 1); // Update vector label=1. EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, 1, 10), 0); // Add a new vector @@ -3941,9 +3973,9 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalSingle) { // Add another one EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, n + 1, n + 1), 1); // Remove vector label=0, it is copied to backend index. - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, 0), 2); + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, 0), 1); // Remove the last vector copied to backend index - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 1), 2); + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 1), 1); // Update vector label=2. EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, 2, 20), 0); // Remove vector label=2. @@ -3965,16 +3997,15 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalSingle) { // 0:deleted, 1: 10, 2: deleted, 3:3, ..., n-2:deleted n-1: 10(n-1), n+1: n+1; // total: n-2 vectors and labels ASSERT_EQ(tiered_index->indexLabelCount(), n - 2); - EXPECT_EQ(tiered_index->GetBackendIndex()->indexLabelCount(), n - 5); + // Nothing remains in flat. + EXPECT_EQ(tiered_index->GetBackendIndex()->indexLabelCount(), n - 2); - // We added 3 vectors to the flat index and removed 5 vectors from the backend index. - // Backend index: 0:deleted, 1:deleted, 2:deleted, 3:3, ..., n-2:deleted, n-1:deleted; - // total: n-5 - EXPECT_EQ(tiered_index->GetBackendIndex()->indexSize(), n); - ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 5); - // Frontend index: 1:10, n-1:10(n-1), n+1:n+1 - ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 3); - ASSERT_EQ(tiered_index->indexSize(), n + tiered_index->GetFlatIndex()->indexSize()); + EXPECT_EQ(tiered_index->GetBackendIndex()->indexSize(), n - 2); + ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 0); + // Flat buffer fully drained by the insert jobs. + ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); + ASSERT_EQ(tiered_index->indexSize(), tiered_index->GetBackendIndex()->indexSize() + + tiered_index->GetFlatIndex()->indexSize()); double abs_err = 1e-2; // Allow a larger relative error for quantization. TEST_DATA_T expected_vector[dim]; @@ -3987,7 +4018,7 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalSingle) { GenerateVector(expected_vector, dim, 0); ASSERT_TRUE(std::isnan(tiered_index->getDistanceFrom_Unsafe(n - 2, expected_vector))); - // Vector label=1, with value 10 should be in the flat index. + // Vector label=1 was updated to value 10; its latest value is retrievable from the index. GenerateVector(expected_vector, dim, 10); ASSERT_NEAR(tiered_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); @@ -4066,22 +4097,24 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalMulti) { // update job paused, we have vectors 0-(n-1) in the index, let's do index modifications - // Remove vector label=n-2, it is copied to backend index. - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 2), 2); + // Remove vector label=n-2. At the pause it lives only in the backend + //, so a single vector is removed. + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 2), 1); // Add one more vector label=1. EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, 1, 10), 1); // Add a new vector EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, n, n), 1); // Add another one EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, n + 1, n + 1), 1); - // Remove vector label=0, it is copied to backend index. - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, 0), 2); - // Remove the last vector copied to backend index - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 1), 2); + // Remove vector label=0, only in the backend at the pause. + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, 0), 1); + // Remove the last vector, only in the backend at the pause. + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n - 1), 1); // Add one more vector label=2. EXPECT_EQ(GenerateAndAddVector(tiered_index, dim, 2, 20), 1); - // Remove vector label=2: for multi: old is copied to backend , old + new are in flat - EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, 2), 3); + // Remove vector label=2: for multi both the original (2) and the new (20) copies sit in + // the flat buffer at this point, so 2 vectors are removed. + EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, 2), 2); // Remove vector label=n - in flat only EXPECT_EQ(VecSimIndex_DeleteVector(tiered_index, n), 1); // Add vector (n-1) again @@ -4099,15 +4132,16 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalMulti) { // 0: deleted, 1: (1,10), 2: deleted, 3:3, ..., n-2: deleted n-1: 10(n-1), n+1: n+1; // total: n-2 labels, n-1 vectors ASSERT_EQ(tiered_index->indexLabelCount(), n - 2); - EXPECT_EQ(tiered_index->GetBackendIndex()->indexLabelCount(), n - 4); + // In the async-insert design the per-vector insert jobs drain every surviving vector from + // the flat buffer into the backend, so nothing remains in flat. + EXPECT_EQ(tiered_index->GetBackendIndex()->indexLabelCount(), n - 2); - // We added 3 vectors to the flat index and removed 4 vectors from the backend index. - // Backend index: 0:deleted, 1:1, 2:deleted, 3:3, ..., n-2:deleted, n-1:deleted; total: n-4 - EXPECT_EQ(tiered_index->GetBackendIndex()->indexSize(), n); - ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 4); - // Frontend index: 1:10, n-1:10(n-1), n+1:n+1 - ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 3); - ASSERT_EQ(tiered_index->indexSize(), n + tiered_index->GetFlatIndex()->indexSize()); + EXPECT_EQ(tiered_index->GetBackendIndex()->indexSize(), n - 1); + ASSERT_EQ(tiered_index->getNumMarkedDeleted(), 0); + // Flat buffer fully drained by the insert jobs. + ASSERT_EQ(tiered_index->GetFlatIndex()->indexSize(), 0); + ASSERT_EQ(tiered_index->indexSize(), tiered_index->GetBackendIndex()->indexSize() + + tiered_index->GetFlatIndex()->indexSize()); double abs_err = 1e-2; // Allow a larger relative error for quantization. TEST_DATA_T expected_vector[dim]; @@ -4120,11 +4154,11 @@ TYPED_TEST(SVSTieredIndexTestBasic, testSwapJournalMulti) { GenerateVector(expected_vector, dim, 0); ASSERT_TRUE(std::isnan(tiered_index->getDistanceFrom_Unsafe(n - 2, expected_vector))); - // There are 2 vectors labeled "1" with values 1 in backend and 10 in flat. - // We expect the minimal distance for the query 10 to be taken from flat index. + // Label "1" is multi-valued and keeps both of its vectors (values 1 and 10). + // The minimal distance for the query 10 should match the second value. GenerateVector(expected_vector, dim, 10); ASSERT_NEAR(tiered_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); - // And the minimal distance for the query 1.0 to be taken from backend + // And the minimal distance for the query 1.0 should match the first value. GenerateVector(expected_vector, dim, 1); ASSERT_NEAR(tiered_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); @@ -4224,8 +4258,9 @@ TYPED_TEST(SVSTieredIndexTestBasic, testDeletedJournalSingle) { mock_thread_pool.thread_pool_join(); - // Verify that vectors labels: {0, 1, 2, n-1} are marked as deleted in the SVS index. - ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 4); + // The update job is paused before it transfers vectors to the backend, so at the pause the + // backend is still empty. + ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 0); // Verify that the deleted vectors are not accessible. double abs_err = 1e-2; // Allow a larger relative error for quantization. @@ -4242,9 +4277,10 @@ TYPED_TEST(SVSTieredIndexTestBasic, testDeletedJournalSingle) { ASSERT_TRUE(std::isnan(tiered_index->getDistanceFrom_Unsafe(label, expected_vector))); } - // label 1 - updated to 10 but deleted in the SVS index - ASSERT_TRUE(flat_index->isLabelExists(1)); - ASSERT_FALSE(svs_index->isLabelExists(1)); + // label 1 - updated to 10 during the pause. The insert job drains it from the flat buffer + // into the backend, so it now lives in the SVS index (not flat) with its updated value. + ASSERT_FALSE(flat_index->isLabelExists(1)); + ASSERT_TRUE(svs_index->isLabelExists(1)); GenerateVector(expected_vector, dim, 10); ASSERT_NEAR(tiered_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); } @@ -4326,8 +4362,9 @@ TYPED_TEST(SVSTieredIndexTestBasic, testDeletedJournalMulti) { mock_thread_pool.thread_pool_join(); - // Verify that vectors labels: {0, 2, n-1} are marked as deleted in the SVS index. - ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 3); + // The update job is paused before it transfers vectors to the backend, so at the pause the + // backend is still empty. + ASSERT_EQ(tiered_index->GetSVSIndex()->getNumMarkedDeleted(), 0); // Verify that the deleted vectors are not accessible. double abs_err = 1e-2; // Allow a larger relative error for quantization. @@ -4344,13 +4381,14 @@ TYPED_TEST(SVSTieredIndexTestBasic, testDeletedJournalMulti) { ASSERT_TRUE(std::isnan(tiered_index->getDistanceFrom_Unsafe(label, expected_vector))); } - // label 1 - multi-value 1 (in SVS) and 10 (in flat) - ASSERT_TRUE(flat_index->isLabelExists(1)); + // label 1 - multi-value with both vectors (1 and 10). The insert jobs drain the flat buffer + // into the backend, so both values now live in the SVS index and none remain in flat. + ASSERT_FALSE(flat_index->isLabelExists(1)); ASSERT_TRUE(svs_index->isLabelExists(1)); GenerateVector(expected_vector, dim, 1); ASSERT_NEAR(backend_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); GenerateVector(expected_vector, dim, 10); - ASSERT_NEAR(flat_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); + ASSERT_NEAR(backend_index->getDistanceFrom_Unsafe(1, expected_vector), 0, abs_err); } TEST(SVSTieredIndexTest, testThreadPool) {