From b1313a466a883f440976ac60c13805078f7d0ac6 Mon Sep 17 00:00:00 2001 From: ray Date: Mon, 29 Jun 2026 19:47:27 +0800 Subject: [PATCH 01/25] refactor: add quantizer and distance --- src/turbo/quantizer/distance.h | 107 +++++++++++++++++++ src/turbo/quantizer/quantizer.h | 180 ++++++++++++++++++++++++++++++++ 2 files changed, 287 insertions(+) create mode 100644 src/turbo/quantizer/distance.h create mode 100644 src/turbo/quantizer/quantizer.h diff --git a/src/turbo/quantizer/distance.h b/src/turbo/quantizer/distance.h new file mode 100644 index 000000000..bc8af6c1a --- /dev/null +++ b/src/turbo/quantizer/distance.h @@ -0,0 +1,107 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include + +namespace zvec { +namespace turbo { + +//! A callable distance handle bound to a quantized query vector. +//! +//! DistanceImpl owns the quantized query bytes and a dispatched +//! DistanceFunc. Invoking `operator()(candidate)` computes the distance +//! between the stored query and the given candidate vector, which is +//! expected to already be in the same quantized layout. +class DistanceImpl { + public: + DistanceImpl() = default; + + DistanceImpl(DistanceFunc func, std::string quantized_query, size_t dim) + : func_(std::move(func)), + query_storage_(std::move(quantized_query)), + dim_(dim) {} + + DistanceImpl(DistanceFunc func, BatchDistanceFunc batch_func, + std::string quantized_query, size_t dim) + : func_(std::move(func)), + batch_func_(std::move(batch_func)), + query_storage_(std::move(quantized_query)), + dim_(dim) {} + + //! Whether the handle is ready to compute distances. + bool valid() const { + return static_cast(func_); + } + + //! Whether a batch distance function is available. + bool batch_valid() const { + return static_cast(batch_func_); + } + + //! Compute the distance between the stored query and `candidate`. + float operator()(const void *candidate) const { + float d = 0.0f; + func_(candidate, query_storage_.data(), dim_, &d); + return d; + } + + //! Compute distances for a batch of `num` candidates against the + //! stored query. Falls back to the scalar path when no batch function + //! is bound. + void batch(const void **candidates, size_t num, float *out) const { + if (batch_func_) { + batch_func_(candidates, query_storage_.data(), num, dim_, out); + return; + } + for (size_t i = 0; i < num; ++i) { + out[i] = 0.0f; + func_(candidates[i], query_storage_.data(), dim_, out + i); + } + } + + //! Access the quantized query bytes (for pairwise helpers). + const std::string &query_storage() const { + return query_storage_; + } + + size_t dim() const { + return dim_; + } + + //! Raw scalar distance function (operates on already-quantized + //! candidates). Useful for pairwise node-vs-node distance where no + //! stored query is involved. + const DistanceFunc &func() const { + return func_; + } + + //! Raw batch distance function. + const BatchDistanceFunc &batch_func() const { + return batch_func_; + } + + private: + DistanceFunc func_{}; + BatchDistanceFunc batch_func_{}; + std::string query_storage_{}; + size_t dim_{0}; +}; + +} // namespace turbo +} // namespace zvec diff --git a/src/turbo/quantizer/quantizer.h b/src/turbo/quantizer/quantizer.h new file mode 100644 index 000000000..c6d1db2bc --- /dev/null +++ b/src/turbo/quantizer/quantizer.h @@ -0,0 +1,180 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include "distance.h" + +using namespace zvec::core; + +namespace zvec { +namespace turbo { + +//! Magic number ('QTZR') stamped at the start of a serialized quantizer blob. +constexpr uint32_t kQuantizerMagic = 0x52545A51u; +//! Current quantizer serialization format version. +constexpr uint16_t kQuantizerSerVersion = 1; + +//! Self-describing, fixed-size header that prefixes every serialized quantizer. +//! The type-specific payload (scalar params, codebook, rotation matrix, ...) +//! follows immediately after this header. +struct QuantizerSerHeader { + uint32_t magic; // kQuantizerMagic + uint16_t version; // kQuantizerSerVersion + uint16_t quant_type; // QuantizeType + uint32_t dim; // original dim (sanity check) + uint32_t metric; // MetricType (sanity check) + uint32_t payload_size; // bytes following the header + uint32_t reserved; // 0, for future use / alignment +}; +static_assert(sizeof(QuantizerSerHeader) == 24, + "QuantizerSerHeader must be 24 bytes"); + +class Quantizer { + public: + typedef std::shared_ptr Pointer; + + Quantizer() {} + virtual ~Quantizer() {} + + //! Initialize quantizer with index metadata and parameters + virtual int init(const IndexMeta &meta, const ailego::Params ¶ms) = 0; + + //! Get the output metadata after initialization + virtual const IndexMeta &meta() const = 0; + + //! Input data type accepted by the quantizer + virtual DataType input_data_type() const = 0; + + //! Data type + virtual QuantizeType type() const { + return type_; + } + + //! Dimensionality of the input vectors + virtual int dim() const = 0; + + //! Train the quantizer with a contiguous batch of data + virtual int train(const void * /*data*/, size_t /*num*/, size_t /*stride*/) { + return IndexError_NotImplemented; + } + + //! Whether the quantizer requires training before use + virtual bool require_train() const = 0; + + //! Train the quantizer with data from an IndexHolder + virtual int train(IndexHolder::Pointer /*holder*/) { + return IndexError_NotImplemented; + } + + //! Byte length of a quantized datapoint vector + virtual size_t quantized_datapoint_vector_length() const = 0; + + //! Byte length of a quantized query vector + virtual size_t quantized_query_vector_length() const = 0; + + //! Quantize a datapoint vector + virtual void quantize_data(const void *input, void *output) const = 0; + + //! Quantize a query vector + virtual void quantize_query(const void *input, void *output) const = 0; + + //! Distance between a quantized datapoint and a quantized query + virtual float calc_distance_dp_query(const void *dp, + const void *query) const = 0; + + //! Batched distance between quantized datapoints and a quantized query + virtual void calc_distance_dp_query_batch(const void *const *dp_list, + int dp_num, const void *query, + float *dist_list) const = 0; + + //! Distance between a quantized datapoint and an unquantized query + virtual float calc_distance_dp_query_unquantized(const void *dp, + const void *query) const = 0; + + //! Batched distance between quantized datapoints and an unquantized query + virtual void calc_distance_dp_query_batch_unquantized( + const void *const *dp_list, int dp_num, const void *query, + float *dist_list) const = 0; + + //! Distance between two quantized datapoints + virtual float calc_distance_dp_dp(const void *dp1, const void *dp2) const = 0; + + //! Quantize a query vector for search + virtual int quantize(const void * /*query*/, const IndexQueryMeta & /*qmeta*/, + std::string * /*out*/, + IndexQueryMeta * /*ometa*/) const { + return IndexError_NotImplemented; + } + + //! Dequantize a result vector back to original format + virtual int dequantize(const void * /*in*/, const IndexQueryMeta & /*qmeta*/, + std::string * /*out*/) const { + return IndexError_NotImplemented; + } + + virtual DistanceImpl distance(const void * /*query*/, + const IndexQueryMeta & /*qmeta*/) const { + return DistanceImpl{}; + } + + //! Serialize quantizer parameters + virtual int serialize(std::string * /*out*/) const { + return IndexError_NotImplemented; + } + + //! Deserialize quantizer parameters + virtual int deserialize(std::string & /*in*/) { + return IndexError_NotImplemented; + } + + //! Deserialize quantizer parameters from a raw, possibly mmap-backed buffer + //! (zero-copy entry point for large payloads such as codebooks/matrices). + virtual int deserialize(const void * /*data*/, size_t /*len*/) { + return IndexError_NotImplemented; + } + + protected: + //! Map a metric name (e.g. "SquaredEuclidean", "Cosine", + //! "InnerProduct", "MipsSquaredEuclidean") to its MetricType. + static MetricType metric_from_name(const std::string &name) { + if (name == "SquaredEuclidean") { + return MetricType::kSquaredEuclidean; + } + if (name == "Cosine") { + return MetricType::kCosine; + } + if (name == "InnerProduct") { + return MetricType::kInnerProduct; + } + if (name == "MipsSquaredEuclidean") { + return MetricType::kMipsSquaredEuclidean; + } + return MetricType::kUnknown; + } + + QuantizeType type_{QuantizeType::kDefault}; + uint32_t extra_meta_size_{0}; +}; + +} // namespace turbo +} // namespace zvec From 45d9b54a10259cfe988dfbadc39bc516dbef5d19 Mon Sep 17 00:00:00 2001 From: ray Date: Tue, 30 Jun 2026 14:16:10 +0800 Subject: [PATCH 02/25] fix: add turo quantizer --- src/turbo/CMakeLists.txt | 4 +- .../record_quantized_int8/common.h | 0 .../record_quantized_int8/cosine.cc | 0 .../record_quantized_int8/cosine.h | 0 .../squared_euclidean.cc | 0 .../record_quantized_int8/squared_euclidean.h | 0 .../avx512_vnni/uniform_int8/quantize.cc | 0 .../avx512_vnni/uniform_int8/quantize.h | 0 .../uniform_int8/squared_euclidean.cc | 0 .../uniform_int8/squared_euclidean.h | 0 tests/turbo/turbo_fp32_quantizer_test.cc | 83 +++++++++++++++++++ 11 files changed, 85 insertions(+), 2 deletions(-) rename src/turbo/{ => distance}/avx512_vnni/record_quantized_int8/common.h (100%) rename src/turbo/{ => distance}/avx512_vnni/record_quantized_int8/cosine.cc (100%) rename src/turbo/{ => distance}/avx512_vnni/record_quantized_int8/cosine.h (100%) rename src/turbo/{ => distance}/avx512_vnni/record_quantized_int8/squared_euclidean.cc (100%) rename src/turbo/{ => distance}/avx512_vnni/record_quantized_int8/squared_euclidean.h (100%) rename src/turbo/{ => distance}/avx512_vnni/uniform_int8/quantize.cc (100%) rename src/turbo/{ => distance}/avx512_vnni/uniform_int8/quantize.h (100%) rename src/turbo/{ => distance}/avx512_vnni/uniform_int8/squared_euclidean.cc (100%) rename src/turbo/{ => distance}/avx512_vnni/uniform_int8/squared_euclidean.h (100%) create mode 100644 tests/turbo/turbo_fp32_quantizer_test.cc diff --git a/src/turbo/CMakeLists.txt b/src/turbo/CMakeLists.txt index 9cbb2fac7..fcf9dc431 100644 --- a/src/turbo/CMakeLists.txt +++ b/src/turbo/CMakeLists.txt @@ -19,7 +19,7 @@ file(GLOB_RECURSE ALL_SRCS *.cc *.c *.h) # subdirectory). if(NOT ANDROID AND AUTO_DETECT_ARCH) if (HOST_ARCH MATCHES "^(x86|x64)$") - file(GLOB_RECURSE AVX512_VNNI_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/avx512_vnni/*.cc) + file(GLOB_RECURSE AVX512_VNNI_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/distance/avx512_vnni/*.cc) set_source_files_properties( ${AVX512_VNNI_SRCS} PROPERTIES @@ -32,5 +32,5 @@ cc_library( NAME zvec_turbo STATIC STRICT PACKED SRCS ${ALL_SRCS} LIBS zvec_ailego - INCS ${CMAKE_CURRENT_SOURCE_DIR} ${PROJECT_ROOT_DIR}/src/include + INCS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/distance ${PROJECT_ROOT_DIR}/src/include ) diff --git a/src/turbo/avx512_vnni/record_quantized_int8/common.h b/src/turbo/distance/avx512_vnni/record_quantized_int8/common.h similarity index 100% rename from src/turbo/avx512_vnni/record_quantized_int8/common.h rename to src/turbo/distance/avx512_vnni/record_quantized_int8/common.h diff --git a/src/turbo/avx512_vnni/record_quantized_int8/cosine.cc b/src/turbo/distance/avx512_vnni/record_quantized_int8/cosine.cc similarity index 100% rename from src/turbo/avx512_vnni/record_quantized_int8/cosine.cc rename to src/turbo/distance/avx512_vnni/record_quantized_int8/cosine.cc diff --git a/src/turbo/avx512_vnni/record_quantized_int8/cosine.h b/src/turbo/distance/avx512_vnni/record_quantized_int8/cosine.h similarity index 100% rename from src/turbo/avx512_vnni/record_quantized_int8/cosine.h rename to src/turbo/distance/avx512_vnni/record_quantized_int8/cosine.h diff --git a/src/turbo/avx512_vnni/record_quantized_int8/squared_euclidean.cc b/src/turbo/distance/avx512_vnni/record_quantized_int8/squared_euclidean.cc similarity index 100% rename from src/turbo/avx512_vnni/record_quantized_int8/squared_euclidean.cc rename to src/turbo/distance/avx512_vnni/record_quantized_int8/squared_euclidean.cc diff --git a/src/turbo/avx512_vnni/record_quantized_int8/squared_euclidean.h b/src/turbo/distance/avx512_vnni/record_quantized_int8/squared_euclidean.h similarity index 100% rename from src/turbo/avx512_vnni/record_quantized_int8/squared_euclidean.h rename to src/turbo/distance/avx512_vnni/record_quantized_int8/squared_euclidean.h diff --git a/src/turbo/avx512_vnni/uniform_int8/quantize.cc b/src/turbo/distance/avx512_vnni/uniform_int8/quantize.cc similarity index 100% rename from src/turbo/avx512_vnni/uniform_int8/quantize.cc rename to src/turbo/distance/avx512_vnni/uniform_int8/quantize.cc diff --git a/src/turbo/avx512_vnni/uniform_int8/quantize.h b/src/turbo/distance/avx512_vnni/uniform_int8/quantize.h similarity index 100% rename from src/turbo/avx512_vnni/uniform_int8/quantize.h rename to src/turbo/distance/avx512_vnni/uniform_int8/quantize.h diff --git a/src/turbo/avx512_vnni/uniform_int8/squared_euclidean.cc b/src/turbo/distance/avx512_vnni/uniform_int8/squared_euclidean.cc similarity index 100% rename from src/turbo/avx512_vnni/uniform_int8/squared_euclidean.cc rename to src/turbo/distance/avx512_vnni/uniform_int8/squared_euclidean.cc diff --git a/src/turbo/avx512_vnni/uniform_int8/squared_euclidean.h b/src/turbo/distance/avx512_vnni/uniform_int8/squared_euclidean.h similarity index 100% rename from src/turbo/avx512_vnni/uniform_int8/squared_euclidean.h rename to src/turbo/distance/avx512_vnni/uniform_int8/squared_euclidean.h diff --git a/tests/turbo/turbo_fp32_quantizer_test.cc b/tests/turbo/turbo_fp32_quantizer_test.cc new file mode 100644 index 000000000..40165a5d3 --- /dev/null +++ b/tests/turbo/turbo_fp32_quantizer_test.cc @@ -0,0 +1,83 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include "zvec/core/framework/index_factory.h" + +using namespace zvec; +using namespace zvec::core; +using namespace zvec::ailego; + +TEST(Fp32Quantizer, General) { + std::mt19937 gen(15583); + std::uniform_real_distribution dist(0.0, 1.0); + + const size_t COUNT = 10000; + const size_t DIMENSION = 12; + + IndexMeta meta; + meta.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION); + meta.set_metric("Cosine", 0, Params()); + + auto quantizer = IndexFactory::CreateQuantizer("Fp32Quantizer"); + ASSERT_TRUE(quantizer); + zvec::ailego::Params params; + ASSERT_EQ(0u, quantizer->init(meta, params)); + + auto holder = + std::make_shared>( + DIMENSION); + for (size_t i = 0; i < COUNT; ++i) { + zvec::ailego::NumericalVector vec(DIMENSION); + for (size_t j = 0; j < DIMENSION; ++j) { + vec[j] = dist(gen); + } + holder->emplace(i + 1, vec); + } + EXPECT_EQ(COUNT, holder->count()); + EXPECT_EQ(IndexMeta::DataType::DT_FP32, holder->data_type()); + + ASSERT_EQ(0u, quantizer->train(holder)); + + auto iter = holder->create_iterator(); + std::string quant_buffer; + std::string dequant_buffer; + + for (; iter->is_valid(); iter->next()) { + EXPECT_TRUE(iter->data()); + + IndexQueryMeta qmeta; + quant_buffer.clear(); + EXPECT_EQ(0, quantizer->quantize( + iter->data(), + IndexQueryMeta(holder->data_type(), holder->dimension()), + &quant_buffer, &qmeta)); + EXPECT_EQ(IndexMeta::DataType::DT_FP32, qmeta.data_type()); + EXPECT_EQ(holder->dimension(), qmeta.dimension()); + + dequant_buffer.clear(); + EXPECT_EQ( + 0, quantizer->dequantize(quant_buffer.data(), qmeta, &dequant_buffer)); + + const float *original_data = reinterpret_cast(iter->data()); + const float *dequantize_data = + reinterpret_cast(dequant_buffer.data()); + for (size_t i = 0; i < holder->dimension(); ++i) { + EXPECT_NEAR(original_data[i], dequantize_data[i], 1e-3); + } + } +} \ No newline at end of file From e82d73f343e75680ba7c66710d91f022c462dd3d Mon Sep 17 00:00:00 2001 From: ray Date: Tue, 30 Jun 2026 15:26:47 +0800 Subject: [PATCH 03/25] refactor: add quantizer --- src/core/framework/index_factory.cc | 13 ++++++++++++ .../zvec/core/framework/index_factory.h | 19 ++++++++++++++++++ src/include/zvec/turbo/turbo.h | 20 +++++++++++++++++++ tests/CMakeLists.txt | 1 + tests/turbo/CMakeLists.txt | 14 +++++++++++++ 5 files changed, 67 insertions(+) create mode 100644 tests/turbo/CMakeLists.txt diff --git a/src/core/framework/index_factory.cc b/src/core/framework/index_factory.cc index 69fe0e98d..e93f57bc7 100644 --- a/src/core/framework/index_factory.cc +++ b/src/core/framework/index_factory.cc @@ -257,5 +257,18 @@ std::vector IndexFactory::AllRefiners(void) { return ailego::Factory::Classes(); } +std::shared_ptr IndexFactory::CreateQuantizer( + const std::string &name) { + return ailego::Factory::MakeShared(name.c_str()); +} + +bool IndexFactory::HasQuantizer(const std::string &name) { + return ailego::Factory::Has(name.c_str()); +} + +std::vector IndexFactory::AllQuantizers(void) { + return ailego::Factory::Classes(); +} + } // namespace core } // namespace zvec diff --git a/src/include/zvec/core/framework/index_factory.h b/src/include/zvec/core/framework/index_factory.h index d891eaa5a..00e77894c 100644 --- a/src/include/zvec/core/framework/index_factory.h +++ b/src/include/zvec/core/framework/index_factory.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include #include @@ -167,6 +168,16 @@ struct IndexFactory { //! Retrieve all refiner classes static std::vector AllRefiners(void); + + //! Create a quantizer by name + static std::shared_ptr CreateQuantizer( + const std::string &name); + + //! Test if the quantizer exists + static bool HasQuantizer(const std::string &name); + + //! Retrieve all quantizer classes + static std::vector AllQuantizers(void); }; //! Register Index Metric @@ -283,5 +294,13 @@ struct IndexFactory { #define INDEX_FACTORY_REGISTER_REFINER(__IMPL__, ...) \ INDEX_FACTORY_REGISTER_REFINER_ALIAS(__IMPL__, __IMPL__, ##__VA_ARGS__) +//! Register Quantizer +#define INDEX_FACTORY_REGISTER_QUANTIZER_ALIAS(__NAME__, __IMPL__, ...) \ + AILEGO_FACTORY_REGISTER(__NAME__, turbo::Quantizer, __IMPL__, ##__VA_ARGS__) + +//! Register Quantizer +#define INDEX_FACTORY_REGISTER_QUANTIZER(__IMPL__, ...) \ + INDEX_FACTORY_REGISTER_QUANTIZER_ALIAS(__IMPL__, __IMPL__, ##__VA_ARGS__) + } // namespace core } // namespace zvec diff --git a/src/include/zvec/turbo/turbo.h b/src/include/zvec/turbo/turbo.h index 2fbf6d680..df353bded 100644 --- a/src/include/zvec/turbo/turbo.h +++ b/src/include/zvec/turbo/turbo.h @@ -36,18 +36,38 @@ using UniformQuantizeFunc = void (*)(const float *in, size_t dim, float scale, enum class MetricType { kSquaredEuclidean, kCosine, + kInnerProduct, kMipsSquaredEuclidean, kUnknown, }; enum class DataType { + kInt4, kInt8, + kFp16, + kFp32, kUnknown, }; enum class QuantizeType { kDefault, kUniform, + kRecord, + kFp16, + kFp32, + kPQ, + kRabit +}; + +enum class CpuArchType { + kAuto, + kScalar, + kSSE, + kAVX, + kAVX2, + kAVX512, + kAVX512VNNI, + kAVX512FP16 }; DistanceFunc get_distance_func(MetricType metric_type, DataType data_type, diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7be2294dd..e3b54ee24 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -7,6 +7,7 @@ include_directories(${PROJECT_ROOT_DIR}) cc_directories(ailego) cc_directories(db) cc_directories(core) +cc_directories(turbo) if(BUILD_C_BINDINGS) cc_directories(c) endif() diff --git a/tests/turbo/CMakeLists.txt b/tests/turbo/CMakeLists.txt new file mode 100644 index 000000000..8a3527d41 --- /dev/null +++ b/tests/turbo/CMakeLists.txt @@ -0,0 +1,14 @@ +include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake) + +file(GLOB_RECURSE ALL_TEST_SRCS *_test.cc) + +foreach(CC_SRCS ${ALL_TEST_SRCS}) + get_filename_component(CC_TARGET ${CC_SRCS} NAME_WE) + cc_gtest( + NAME ${CC_TARGET} + STRICT + LIBS zvec_ailego core_framework core_metric core_quantizer zvec_turbo + SRCS ${CC_SRCS} + INCS . ${PROJECT_ROOT_DIR}/src/core/ ${PROJECT_ROOT_DIR}/src/turbo/ + ) +endforeach() \ No newline at end of file From 829f53e198aa092f0a62ee6f6da2e56b11e4b894 Mon Sep 17 00:00:00 2001 From: ray Date: Tue, 30 Jun 2026 15:29:46 +0800 Subject: [PATCH 04/25] refactor: update meta --- src/core/framework/index_meta.cc | 4 +- src/include/zvec/core/framework/index_meta.h | 50 +++++++++++++++++++- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/core/framework/index_meta.cc b/src/core/framework/index_meta.cc index 11d54cb63..d0eadb02d 100644 --- a/src/core/framework/index_meta.cc +++ b/src/core/framework/index_meta.cc @@ -30,7 +30,8 @@ struct IndexMetaFormatHeader { uint32_t space_id; uint32_t attachment_offset; uint32_t attachment_size; - uint8_t reserved_[4092]; + uint32_t extra_meta_size; + uint8_t reserved_[4088]; }; static_assert(sizeof(IndexMetaFormatHeader) % 32 == 0, @@ -47,6 +48,7 @@ void IndexMeta::serialize(std::string *out) const { format.dimension = dimension_; format.unit_size = unit_size_; format.space_id = space_id_; + format.extra_meta_size = extra_meta_size_; if (!metric_name_.empty()) { ailego::Params item; diff --git a/src/include/zvec/core/framework/index_meta.h b/src/include/zvec/core/framework/index_meta.h index 3a09aaefb..c6c5137db 100644 --- a/src/include/zvec/core/framework/index_meta.h +++ b/src/include/zvec/core/framework/index_meta.h @@ -40,6 +40,7 @@ class IndexMeta { DT_BINARY64 = 8, }; + /*! Major Orders */ enum MajorOrder { @@ -77,6 +78,7 @@ class IndexMeta { dimension_(rhs.dimension_), unit_size_(rhs.unit_size_), element_size_(rhs.element_size_), + extra_meta_size_(rhs.extra_meta_size_), space_id_(rhs.space_id_), metric_revision_(rhs.metric_revision_), converter_revision_(rhs.converter_revision_), @@ -112,6 +114,7 @@ class IndexMeta { dimension_(rhs.dimension_), unit_size_(rhs.unit_size_), element_size_(rhs.element_size_), + extra_meta_size_(rhs.extra_meta_size_), space_id_(rhs.space_id_), metric_revision_(rhs.metric_revision_), converter_revision_(rhs.converter_revision_), @@ -173,6 +176,7 @@ class IndexMeta { searcher_params_ = std::move(rhs.searcher_params_); streamer_params_ = std::move(rhs.streamer_params_); attributes_ = std::move(rhs.attributes_); + extra_meta_size_ = rhs.extra_meta_size_; return *this; } @@ -211,6 +215,7 @@ class IndexMeta { searcher_params_ = std::move(rhs.searcher_params_); streamer_params_ = std::move(rhs.streamer_params_); attributes_ = std::move(rhs.attributes_); + extra_meta_size_ = rhs.extra_meta_size_; return *this; } @@ -249,6 +254,7 @@ class IndexMeta { searcher_params_.clear(); streamer_params_.clear(); attributes_.clear(); + extra_meta_size_ = 0; } //! Retrieve major order information @@ -281,6 +287,11 @@ class IndexMeta { return element_size_; } + //! Retrieve extra meta size in bytes + uint32_t extra_meta_size(void) const { + return extra_meta_size_; + } + //! Retrieve space id uint64_t space_id(void) const { return space_id_; @@ -451,6 +462,11 @@ class IndexMeta { this->set_meta(data_type, UnitSizeof(data_type), dim); } + //! Set extra meta size + void set_extra_meta_size(uint32_t size) { + extra_meta_size_ = size; + } + //! Set information of metric template void set_metric(TName &&name, uint32_t rev, TParams &¶ms) { @@ -586,6 +602,7 @@ class IndexMeta { uint32_t dimension_{0}; uint32_t unit_size_{0}; uint32_t element_size_{0}; + uint32_t extra_meta_size_{0}; uint64_t space_id_{0}; uint32_t metric_revision_{0}; uint32_t converter_revision_{0}; @@ -632,6 +649,19 @@ class IndexQueryMeta { unit_size_(unit), element_size_(IndexMeta::ElementSizeof(data_type, unit, dim)) {} + //! Constructor + IndexQueryMeta(IndexMeta::MetaType meta_type, IndexMeta::DataType data_type, + uint32_t unit, uint32_t dim, uint32_t quantize_type, + uint32_t extra_meta_size) + : meta_type_(meta_type), + data_type_(data_type), + dimension_(dim), + unit_size_(unit), + quantize_type_(quantize_type), + extra_meta_size_(extra_meta_size), + element_size_(IndexMeta::ElementSizeof(data_type, unit, dim) + + extra_meta_size_) {} + //! Constructor IndexQueryMeta(IndexMeta::DataType data_type, uint32_t dim) : IndexQueryMeta{IndexMeta::MetaType::MT_DENSE, data_type, @@ -676,7 +706,8 @@ class IndexQueryMeta { //! Set dimension of feature void set_dimension(uint32_t dim) { dimension_ = dim; - element_size_ = IndexMeta::ElementSizeof(data_type_, unit_size_, dim); + element_size_ = IndexMeta::ElementSizeof(data_type_, unit_size_, dim) + + extra_meta_size_; } //! Set meta type @@ -694,7 +725,8 @@ class IndexQueryMeta { data_type_ = data_type; dimension_ = dim; unit_size_ = unit; - element_size_ = IndexMeta::ElementSizeof(data_type, unit, dim); + element_size_ = + IndexMeta::ElementSizeof(data_type, unit, dim) + extra_meta_size_; } //! Set meta information of feature @@ -702,11 +734,25 @@ class IndexQueryMeta { this->set_meta(data_type, IndexMeta::UnitSizeof(data_type), dim); } + //! Set meta information of feature with quantize type and extra meta size + void set_meta(IndexMeta::DataType data_type, uint32_t dim, + uint32_t quantize_type, uint32_t extra_meta_size) { + data_type_ = data_type; + dimension_ = dim; + unit_size_ = IndexMeta::UnitSizeof(data_type); + quantize_type_ = quantize_type; + extra_meta_size_ = extra_meta_size; + element_size_ = + IndexMeta::ElementSizeof(data_type, unit_size_, dim) + extra_meta_size_; + } + private: IndexMeta::MetaType meta_type_{IndexMeta::MetaType::MT_DENSE}; IndexMeta::DataType data_type_{IndexMeta::DataType::DT_UNDEFINED}; uint32_t dimension_{0}; uint32_t unit_size_{0}; + uint32_t quantize_type_{0}; + uint32_t extra_meta_size_{0}; uint32_t element_size_{0}; }; From f08463011b0af38ecde1d987d9279a78f247aefe Mon Sep 17 00:00:00 2001 From: ray Date: Tue, 30 Jun 2026 15:46:13 +0800 Subject: [PATCH 05/25] refactor: update meta --- src/turbo/quantizer/quantizer.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/turbo/quantizer/quantizer.h b/src/turbo/quantizer/quantizer.h index c6d1db2bc..572d372b5 100644 --- a/src/turbo/quantizer/quantizer.h +++ b/src/turbo/quantizer/quantizer.h @@ -24,11 +24,11 @@ #include #include "distance.h" -using namespace zvec::core; - namespace zvec { namespace turbo { +using namespace zvec::core; + //! Magic number ('QTZR') stamped at the start of a serialized quantizer blob. constexpr uint32_t kQuantizerMagic = 0x52545A51u; //! Current quantizer serialization format version. From c3c4ca35c0d1c7389c9c96206a7527be97bf3596 Mon Sep 17 00:00:00 2001 From: ray Date: Tue, 30 Jun 2026 16:22:51 +0800 Subject: [PATCH 06/25] refactor: add fp32 quantizer --- src/include/zvec/turbo/turbo.h | 15 +- .../fp32_quantizer/fp32_quantizer.cc | 161 ++++++++++++++++++ .../quantizer/fp32_quantizer/fp32_quantizer.h | 127 ++++++++++++++ src/turbo/turbo.cc | 22 ++- 4 files changed, 312 insertions(+), 13 deletions(-) create mode 100644 src/turbo/quantizer/fp32_quantizer/fp32_quantizer.cc create mode 100644 src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h diff --git a/src/include/zvec/turbo/turbo.h b/src/include/zvec/turbo/turbo.h index df353bded..2eff572d5 100644 --- a/src/include/zvec/turbo/turbo.h +++ b/src/include/zvec/turbo/turbo.h @@ -71,15 +71,16 @@ enum class CpuArchType { }; DistanceFunc get_distance_func(MetricType metric_type, DataType data_type, - QuantizeType quantize_type); + QuantizeType quantize_type, + CpuArchType cpu_arch_type = CpuArchType::kAuto); -BatchDistanceFunc get_batch_distance_func(MetricType metric_type, - DataType data_type, - QuantizeType quantize_type); +BatchDistanceFunc get_batch_distance_func( + MetricType metric_type, DataType data_type, QuantizeType quantize_type, + CpuArchType cpu_arch_type = CpuArchType::kAuto); -QueryPreprocessFunc get_query_preprocess_func(MetricType metric_type, - DataType data_type, - QuantizeType quantize_type); +QueryPreprocessFunc get_query_preprocess_func( + MetricType metric_type, DataType data_type, QuantizeType quantize_type, + CpuArchType cpu_arch_type = CpuArchType::kAuto); // Returns the SIMD kernel for the uniform quantizer on the current CPU for // the given output data_type, or nullptr if no SIMD implementation is diff --git a/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.cc b/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.cc new file mode 100644 index 000000000..efd4e951e --- /dev/null +++ b/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.cc @@ -0,0 +1,161 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "quantizer/fp32_quantizer/fp32_quantizer.h" +#include +#include +#include +#include +#include +#include +#include "core/quantizer/record_quantizer.h" + +namespace zvec { +namespace turbo { + +int Fp32Quantizer::init(const IndexMeta &meta, + const ailego::Params & /*params*/) { + meta_ = meta; + + meta_.set_meta(IndexMeta::DataType::DT_FP32, meta.dimension()); + + auto metric_name = meta.metric_name(); + if (metric_name == "Cosine") { + extra_meta_size_ = EXTRA_META_SIZE_COSINE; + meta_.set_extra_meta_size(extra_meta_size_); + } + + // `meta.dimension()` may be either the raw dim (when the caller passes a + // bare meta, e.g. unit tests) or the inflated storage dim (data + extras) + // when an upstream converter such as CosineConverter has already appended + // a norm float and set meta.extra_meta_size(). Distinguish via the input + // meta's extra_meta_size: if it is already set, the dim is inflated and + // we strip the extras to recover the raw dim; otherwise the dim is raw. + if (meta.extra_meta_size() > 0) { + original_dim_ = meta.dimension() - meta.extra_meta_size() / sizeof(float); + } else { + original_dim_ = meta.dimension(); + } + + // Cache the distance dispatch for the new Quantizer interface. + dp_query_func_ = + get_distance_func(metric_from_name(metric_name), DataType::kFp32, + QuantizeType::kDefault, CpuArchType::kAuto); + dp_query_batch_func_ = + get_batch_distance_func(metric_from_name(metric_name), DataType::kFp32, + QuantizeType::kDefault, CpuArchType::kAuto); + + return 0; +} + +int Fp32Quantizer::quantize(const void *query, const IndexQueryMeta &qmeta, + std::string *out, IndexQueryMeta *ometa) const { + if (qmeta.unit_size() != sizeof(float)) { + return IndexError_Unsupported; + } + + // qmeta.dimension() may be the inflated (data + extras) dimension when the + // caller uses meta_.dimension() directly (e.g. HnswDistCalculator). Use the + // raw original dim we recorded at init() to avoid over-reading the query. + size_t raw_dim = (original_dim_ != 0 && qmeta.dimension() >= original_dim_) + ? original_dim_ + : qmeta.dimension(); + size_t byte_size = raw_dim * sizeof(float); + out->resize(byte_size); + std::memcpy(&(*out)[0], query, byte_size); + + *ometa = qmeta; + ometa->set_meta(IndexMeta::DataType::DT_FP32, raw_dim, + static_cast(type_), extra_meta_size_); + + return 0; +} + +int Fp32Quantizer::dequantize(const void *in, const IndexQueryMeta &qmeta, + std::string *out) const { + size_t byte_size = qmeta.dimension() * sizeof(float); + out->resize(byte_size); + std::memcpy(out->data(), in, byte_size); + return 0; +} + +DistanceImpl Fp32Quantizer::distance(const void *query, + const IndexQueryMeta &qmeta) const { + auto metric = metric_from_name(meta_.metric_name()); + auto func = get_distance_func(metric, DataType::kFp32, QuantizeType::kDefault, + CpuArchType::kAuto); + if (!func) { + return DistanceImpl{}; + } + auto batch_func = get_batch_distance_func( + metric, DataType::kFp32, QuantizeType::kDefault, CpuArchType::kAuto); + + // The query is assumed to be already quantized — copy it directly. + std::string quantized_query(static_cast(query), + qmeta.element_size()); + return DistanceImpl(std::move(func), std::move(batch_func), + std::move(quantized_query), original_dim_); +} + +void Fp32Quantizer::quantize_one(const void *input, void *output) const { + std::memcpy(output, input, + static_cast(original_dim_) * sizeof(float)); +} + +float Fp32Quantizer::calc_distance_dp_query(const void *dp, + const void *query) const { + float d = 0.0f; + if (dp_query_func_) { + dp_query_func_(dp, query, original_dim_, &d); + } + return d; +} + +void Fp32Quantizer::calc_distance_dp_query_batch(const void *const *dp_list, + int dp_num, const void *query, + float *dist_list) const { + if (dp_query_batch_func_) { + dp_query_batch_func_(const_cast(dp_list), query, + static_cast(dp_num), original_dim_, dist_list); + return; + } + for (int i = 0; i < dp_num; ++i) { + dist_list[i] = calc_distance_dp_query(dp_list[i], query); + } +} + +float Fp32Quantizer::calc_distance_dp_query_unquantized( + const void *dp, const void *query) const { + std::string buf(quantized_length(), '\0'); + quantize_one(query, &buf[0]); + return calc_distance_dp_query(dp, buf.data()); +} + +void Fp32Quantizer::calc_distance_dp_query_batch_unquantized( + const void *const *dp_list, int dp_num, const void *query, + float *dist_list) const { + std::string buf(quantized_length(), '\0'); + quantize_one(query, &buf[0]); + calc_distance_dp_query_batch(dp_list, dp_num, buf.data(), dist_list); +} + +float Fp32Quantizer::calc_distance_dp_dp(const void *dp1, + const void *dp2) const { + return calc_distance_dp_query(dp1, dp2); +} + +INDEX_FACTORY_REGISTER_QUANTIZER(Fp32Quantizer); + +} // namespace turbo +} // namespace zvec diff --git a/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h b/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h new file mode 100644 index 000000000..998a19d68 --- /dev/null +++ b/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h @@ -0,0 +1,127 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include +#include "quantizer/quantizer.h" + +namespace zvec { +namespace turbo { + +using namespace zvec::core; + +class Fp32Quantizer : public Quantizer { + public: + Fp32Quantizer() { + type_ = QuantizeType::kFp32; + } + + virtual ~Fp32Quantizer() {} + + public: + int init(const core::IndexMeta &meta, const ailego::Params ¶ms) override; + + const core::IndexMeta &meta(void) const override { + return meta_; + } + + DataType input_data_type() const override { + return DataType::kFp32; + } + + QuantizeType type() const override { + return type_; + } + + int dim() const override { + return static_cast(original_dim_); + } + + bool require_train() const override { + return false; + } + + int train(core::IndexHolder::Pointer /*holder*/) override { + return 0; + } + + size_t quantized_datapoint_vector_length() const override { + return quantized_length(); + } + + size_t quantized_query_vector_length() const override { + return quantized_length(); + } + + void quantize_data(const void *input, void *output) const override { + quantize_one(input, output); + } + + void quantize_query(const void *input, void *output) const override { + quantize_one(input, output); + } + + float calc_distance_dp_query(const void *dp, + const void *query) const override; + + void calc_distance_dp_query_batch(const void *const *dp_list, int dp_num, + const void *query, + float *dist_list) const override; + + float calc_distance_dp_query_unquantized(const void *dp, + const void *query) const override; + + void calc_distance_dp_query_batch_unquantized( + const void *const *dp_list, int dp_num, const void *query, + float *dist_list) const override; + + float calc_distance_dp_dp(const void *dp1, const void *dp2) const override; + + int quantize(const void *query, const core::IndexQueryMeta &qmeta, + std::string *out, core::IndexQueryMeta *ometa) const override; + + int dequantize(const void *in, const core::IndexQueryMeta &qmeta, + std::string *out) const override; + + DistanceImpl distance(const void *query, + const core::IndexQueryMeta &qmeta) const override; + + private: + //! Byte length of a quantized vector (raw fp32 data). + size_t quantized_length() const { + return static_cast(original_dim_) * sizeof(float); + } + + //! Quantize a single fp32 vector into a caller-provided buffer of + //! quantized_length() bytes. + void quantize_one(const void *input, void *output) const; + + static constexpr uint32_t EXTRA_META_SIZE_COSINE = 4; + + IndexMeta meta_{}; + uint32_t original_dim_{0}; + IndexMeta::DataType data_type_{}; + + //! Cached distance dispatch (bound in init()). + DistanceFunc dp_query_func_{}; + BatchDistanceFunc dp_query_batch_func_{}; +}; + + +} // namespace turbo +} // namespace zvec diff --git a/src/turbo/turbo.cc b/src/turbo/turbo.cc index adc9b785e..11a7a43cc 100644 --- a/src/turbo/turbo.cc +++ b/src/turbo/turbo.cc @@ -22,10 +22,13 @@ namespace zvec::turbo { DistanceFunc get_distance_func(MetricType metric_type, DataType data_type, - QuantizeType quantize_type) { + QuantizeType quantize_type, + CpuArchType cpu_arch_type) { if (data_type == DataType::kInt8) { if (quantize_type == QuantizeType::kDefault) { - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) { + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI && + (cpu_arch_type == CpuArchType::kAuto || + cpu_arch_type == CpuArchType::kAVX512VNNI)) { if (metric_type == MetricType::kSquaredEuclidean) { return avx512_vnni::squared_euclidean_int8_distance; } @@ -47,10 +50,13 @@ DistanceFunc get_distance_func(MetricType metric_type, DataType data_type, BatchDistanceFunc get_batch_distance_func(MetricType metric_type, DataType data_type, - QuantizeType quantize_type) { + QuantizeType quantize_type, + CpuArchType cpu_arch_type) { if (data_type == DataType::kInt8) { if (quantize_type == QuantizeType::kDefault) { - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) { + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI && + (cpu_arch_type == CpuArchType::kAuto || + cpu_arch_type == CpuArchType::kAVX512VNNI)) { if (metric_type == MetricType::kSquaredEuclidean) { return avx512_vnni::squared_euclidean_int8_batch_distance; } @@ -67,15 +73,19 @@ BatchDistanceFunc get_batch_distance_func(MetricType metric_type, } } } + return nullptr; } QueryPreprocessFunc get_query_preprocess_func(MetricType metric_type, DataType data_type, - QuantizeType quantize_type) { + QuantizeType quantize_type, + CpuArchType cpu_arch_type) { if (data_type == DataType::kInt8) { if (quantize_type == QuantizeType::kDefault) { - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) { + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI && + (cpu_arch_type == CpuArchType::kAuto || + cpu_arch_type == CpuArchType::kAVX512VNNI)) { if (metric_type == MetricType::kSquaredEuclidean) { return avx512_vnni::squared_euclidean_int8_query_preprocess; } From ec4ea1fdfbfd6b5240cef161c1820b15f070c1f3 Mon Sep 17 00:00:00 2001 From: ray Date: Tue, 30 Jun 2026 16:35:47 +0800 Subject: [PATCH 07/25] refactor: add quantizer --- src/turbo/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/turbo/CMakeLists.txt b/src/turbo/CMakeLists.txt index fcf9dc431..7ec9c0e5d 100644 --- a/src/turbo/CMakeLists.txt +++ b/src/turbo/CMakeLists.txt @@ -29,7 +29,7 @@ if(NOT ANDROID AND AUTO_DETECT_ARCH) endif() cc_library( - NAME zvec_turbo STATIC STRICT PACKED + NAME zvec_turbo STATIC STRICT ALWAYS_LINK PACKED SRCS ${ALL_SRCS} LIBS zvec_ailego INCS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/distance ${PROJECT_ROOT_DIR}/src/include From e36f0c5495735fba1ba04e82149e9e39fcbac122 Mon Sep 17 00:00:00 2001 From: ray Date: Tue, 30 Jun 2026 17:34:23 +0800 Subject: [PATCH 08/25] refactor: quantizer --- src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h b/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h index 998a19d68..79a7caa09 100644 --- a/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h +++ b/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h @@ -57,7 +57,7 @@ class Fp32Quantizer : public Quantizer { } int train(core::IndexHolder::Pointer /*holder*/) override { - return 0; + return IndexError_NotImplemented; } size_t quantized_datapoint_vector_length() const override { From c1cdd0b8c0fbbf337a85cc17bc4391cf234540e4 Mon Sep 17 00:00:00 2001 From: ray Date: Tue, 30 Jun 2026 19:08:56 +0800 Subject: [PATCH 09/25] refactor: add scalar --- src/turbo/distance/scalar/fp32/cosine.cc | 47 +++++++ src/turbo/distance/scalar/fp32/cosine.h | 30 +++++ .../distance/scalar/fp32/inner_product.cc | 43 +++++++ .../distance/scalar/fp32/inner_product.h | 31 +++++ .../distance/scalar/fp32/squared_euclidean.cc | 41 ++++++ .../distance/scalar/fp32/squared_euclidean.h | 31 +++++ src/turbo/turbo.cc | 33 +++++ tests/turbo/turbo_fp32_quantizer_test.cc | 119 ++++++++++++++++++ 8 files changed, 375 insertions(+) create mode 100644 src/turbo/distance/scalar/fp32/cosine.cc create mode 100644 src/turbo/distance/scalar/fp32/cosine.h create mode 100644 src/turbo/distance/scalar/fp32/inner_product.cc create mode 100644 src/turbo/distance/scalar/fp32/inner_product.h create mode 100644 src/turbo/distance/scalar/fp32/squared_euclidean.cc create mode 100644 src/turbo/distance/scalar/fp32/squared_euclidean.h diff --git a/src/turbo/distance/scalar/fp32/cosine.cc b/src/turbo/distance/scalar/fp32/cosine.cc new file mode 100644 index 000000000..112a6fc80 --- /dev/null +++ b/src/turbo/distance/scalar/fp32/cosine.cc @@ -0,0 +1,47 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "scalar/fp32/cosine.h" +#include + +namespace zvec::turbo::scalar { + +void cosine_fp32_distance(const void *a, const void *b, size_t dim, + float *distance) { + const float *fa = static_cast(a); + const float *fb = static_cast(b); + float dot = 0.0f; + float na = 0.0f; + float nb = 0.0f; + for (size_t i = 0; i < dim; ++i) { + dot += fa[i] * fb[i]; + na += fa[i] * fa[i]; + nb += fb[i] * fb[i]; + } + float denom = std::sqrt(na) * std::sqrt(nb); + if (denom < 1e-12f) { + *distance = 1.0f; + } else { + *distance = 1.0f - dot / denom; + } +} + +void cosine_fp32_batch_distance(const void *const *vectors, const void *query, + size_t n, size_t dim, float *distances) { + for (size_t i = 0; i < n; ++i) { + cosine_fp32_distance(vectors[i], query, dim, &distances[i]); + } +} + +} // namespace zvec::turbo::scalar \ No newline at end of file diff --git a/src/turbo/distance/scalar/fp32/cosine.h b/src/turbo/distance/scalar/fp32/cosine.h new file mode 100644 index 000000000..b5e4f4eee --- /dev/null +++ b/src/turbo/distance/scalar/fp32/cosine.h @@ -0,0 +1,30 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace zvec::turbo::scalar { + +// Compute cosine distance (negative inner product after normalization) between +// a single quantized FP32 vector pair. +void cosine_fp32_distance(const void *a, const void *b, size_t dim, + float *distance); + +// Batch version of cosine_fp32_distance. +void cosine_fp32_batch_distance(const void *const *vectors, const void *query, + size_t n, size_t dim, float *distances); + +} // namespace zvec::turbo::scalar \ No newline at end of file diff --git a/src/turbo/distance/scalar/fp32/inner_product.cc b/src/turbo/distance/scalar/fp32/inner_product.cc new file mode 100644 index 000000000..63a0575a3 --- /dev/null +++ b/src/turbo/distance/scalar/fp32/inner_product.cc @@ -0,0 +1,43 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "scalar/fp32/inner_product.h" + +namespace zvec::turbo::scalar { + +// Compute squared Euclidean distance between a single quantized FP32 +// vector pair. +void inner_product_fp32_distance(const void *a, const void *b, size_t dim, + float *distance) { + const float *m = reinterpret_cast(a); + const float *q = reinterpret_cast(b); + + float sum = 0.0; + for (size_t i = 0; i < dim; ++i) { + sum += static_cast(m[i] * q[i]); + } + + *distance = -sum; +} + +// Batch version of inner_product_fp32_distance. +void inner_product_fp32_batch_distance(const void *const *vectors, + const void *query, size_t n, size_t dim, + float *distances) { + for (size_t i = 0; i < n; ++i) { + inner_product_fp32_distance(vectors[i], query, dim, &distances[i]); + } +} + +} // namespace zvec::turbo::scalar \ No newline at end of file diff --git a/src/turbo/distance/scalar/fp32/inner_product.h b/src/turbo/distance/scalar/fp32/inner_product.h new file mode 100644 index 000000000..d4e03418e --- /dev/null +++ b/src/turbo/distance/scalar/fp32/inner_product.h @@ -0,0 +1,31 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace zvec::turbo::scalar { + +// Compute inner product distance between a single quantized FP32 +// vector pair. +void inner_product_fp32_distance(const void *a, const void *b, size_t dim, + float *distance); + +// Batch version of inner_product_fp32_distance. +void inner_product_fp32_batch_distance(const void *const *vectors, + const void *query, size_t n, size_t dim, + float *distances); + +} // namespace zvec::turbo::scalar diff --git a/src/turbo/distance/scalar/fp32/squared_euclidean.cc b/src/turbo/distance/scalar/fp32/squared_euclidean.cc new file mode 100644 index 000000000..602d8bd7f --- /dev/null +++ b/src/turbo/distance/scalar/fp32/squared_euclidean.cc @@ -0,0 +1,41 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "scalar/fp32/squared_euclidean.h" +#include + +namespace zvec::turbo::scalar { + +void squared_euclidean_fp32_distance(const void *a, const void *b, size_t dim, + float *distance) { + const float *m = reinterpret_cast(a); + const float *q = reinterpret_cast(b); + + float sum = 0.0; + for (size_t i = 0; i < dim; ++i) { + sum += zvec::ailego::MathHelper::SquaredDifference(m[i], q[i]); + } + + *distance = sum; +} + +void squared_euclidean_fp32_batch_distance(const void *const *vectors, + const void *query, size_t n, + size_t dim, float *distances) { + for (size_t i = 0; i < n; ++i) { + squared_euclidean_fp32_distance(vectors[i], query, dim, &distances[i]); + } +} + +} // namespace zvec::turbo::scalar \ No newline at end of file diff --git a/src/turbo/distance/scalar/fp32/squared_euclidean.h b/src/turbo/distance/scalar/fp32/squared_euclidean.h new file mode 100644 index 000000000..bf319c1d2 --- /dev/null +++ b/src/turbo/distance/scalar/fp32/squared_euclidean.h @@ -0,0 +1,31 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace zvec::turbo::scalar { + +// Compute squared euclidean distance between a single quantized FP32 +// vector pair. +void squared_euclidean_fp32_distance(const void *a, const void *b, size_t dim, + float *distance); + +// Batch version of squared euclidean FP32. +void squared_euclidean_fp32_batch_distance(const void *const *vectors, + const void *query, size_t n, + size_t dim, float *distances); + +} // namespace zvec::turbo::scalar diff --git a/src/turbo/turbo.cc b/src/turbo/turbo.cc index 11a7a43cc..80f2ee77d 100644 --- a/src/turbo/turbo.cc +++ b/src/turbo/turbo.cc @@ -18,12 +18,30 @@ #include "avx512_vnni/record_quantized_int8/squared_euclidean.h" #include "avx512_vnni/uniform_int8/quantize.h" #include "avx512_vnni/uniform_int8/squared_euclidean.h" +#include "scalar/fp32/cosine.h" +#include "scalar/fp32/inner_product.h" +#include "scalar/fp32/squared_euclidean.h" namespace zvec::turbo { DistanceFunc get_distance_func(MetricType metric_type, DataType data_type, QuantizeType quantize_type, CpuArchType cpu_arch_type) { + if (data_type == DataType::kFp32) { + if (quantize_type == QuantizeType::kDefault || + quantize_type == QuantizeType::kFp32) { + if (metric_type == MetricType::kCosine) { + return scalar::cosine_fp32_distance; + } + if (metric_type == MetricType::kSquaredEuclidean) { + return scalar::squared_euclidean_fp32_distance; + } + if (metric_type == MetricType::kInnerProduct) { + return scalar::inner_product_fp32_distance; + } + } + return nullptr; + } if (data_type == DataType::kInt8) { if (quantize_type == QuantizeType::kDefault) { if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI && @@ -52,6 +70,21 @@ BatchDistanceFunc get_batch_distance_func(MetricType metric_type, DataType data_type, QuantizeType quantize_type, CpuArchType cpu_arch_type) { + if (data_type == DataType::kFp32) { + if (quantize_type == QuantizeType::kDefault || + quantize_type == QuantizeType::kFp32) { + if (metric_type == MetricType::kCosine) { + return scalar::cosine_fp32_batch_distance; + } + if (metric_type == MetricType::kSquaredEuclidean) { + return scalar::squared_euclidean_fp32_batch_distance; + } + if (metric_type == MetricType::kInnerProduct) { + return scalar::inner_product_fp32_batch_distance; + } + } + return nullptr; + } if (data_type == DataType::kInt8) { if (quantize_type == QuantizeType::kDefault) { if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI && diff --git a/tests/turbo/turbo_fp32_quantizer_test.cc b/tests/turbo/turbo_fp32_quantizer_test.cc index 40165a5d3..c180de4a4 100644 --- a/tests/turbo/turbo_fp32_quantizer_test.cc +++ b/tests/turbo/turbo_fp32_quantizer_test.cc @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include +#include #include #include #include @@ -22,6 +24,18 @@ using namespace zvec; using namespace zvec::core; using namespace zvec::ailego; +// Helper: reference cosine distance between two raw fp32 vectors. +static float reference_cosine(const float *a, const float *b, size_t dim) { + float dot = 0.0f, na = 0.0f, nb = 0.0f; + for (size_t i = 0; i < dim; ++i) { + dot += a[i] * b[i]; + na += a[i] * a[i]; + nb += b[i] * b[i]; + } + float denom = std::sqrt(na) * std::sqrt(nb); + return (denom < 1e-12f) ? 1.0f : 1.0f - dot / denom; +} + TEST(Fp32Quantizer, General) { std::mt19937 gen(15583); std::uniform_real_distribution dist(0.0, 1.0); @@ -80,4 +94,109 @@ TEST(Fp32Quantizer, General) { EXPECT_NEAR(original_data[i], dequantize_data[i], 1e-3); } } +} + +TEST(Fp32Quantizer, Score) { + std::mt19937 gen(42); + std::uniform_real_distribution dist(0.0, 1.0); + + const size_t DIMENSION = 12; + const size_t COUNT = 100; + + IndexMeta meta; + meta.set_meta(IndexMeta::DataType::DT_FP32, DIMENSION); + meta.set_metric("Cosine", 0, Params()); + + auto quantizer = IndexFactory::CreateQuantizer("Fp32Quantizer"); + ASSERT_TRUE(quantizer); + zvec::ailego::Params params; + ASSERT_EQ(0u, quantizer->init(meta, params)); + + // Generate raw vectors and quantize them. + std::vector> raw_vecs(COUNT); + std::vector quant_vecs(COUNT); + for (size_t i = 0; i < COUNT; ++i) { + raw_vecs[i].resize(DIMENSION); + for (size_t j = 0; j < DIMENSION; ++j) { + raw_vecs[i][j] = dist(gen); + } + IndexQueryMeta ometa; + EXPECT_EQ(0, quantizer->quantize( + raw_vecs[i].data(), + IndexQueryMeta(IndexMeta::DataType::DT_FP32, DIMENSION), + &quant_vecs[i], &ometa)); + } + + // --- calc_distance_dp_query (single) --- + for (size_t i = 1; i < COUNT; ++i) { + float d = quantizer->calc_distance_dp_query(quant_vecs[i].data(), + quant_vecs[0].data()); + float expected = + reference_cosine(raw_vecs[i].data(), raw_vecs[0].data(), DIMENSION); + EXPECT_NEAR(d, expected, 1e-4) << "i=" << i; + } + + // --- calc_distance_dp_query_batch --- + { + std::vector dp_list(COUNT - 1); + for (size_t i = 1; i < COUNT; ++i) { + dp_list[i - 1] = quant_vecs[i].data(); + } + std::vector results(COUNT - 1); + quantizer->calc_distance_dp_query_batch( + dp_list.data(), static_cast(dp_list.size()), quant_vecs[0].data(), + results.data()); + + for (size_t i = 0; i < dp_list.size(); ++i) { + float expected = reference_cosine(raw_vecs[i + 1].data(), + raw_vecs[0].data(), DIMENSION); + EXPECT_NEAR(results[i], expected, 1e-4) << "i=" << i; + } + } + + // --- distance() + DistanceImpl (single + batch) --- + { + IndexQueryMeta qmeta(IndexMeta::DataType::DT_FP32, DIMENSION); + auto dist_impl = quantizer->distance(quant_vecs[0].data(), qmeta); + ASSERT_TRUE(dist_impl.valid()); + + for (size_t i = 1; i < COUNT; ++i) { + float d = dist_impl(quant_vecs[i].data()); + float expected = + reference_cosine(raw_vecs[0].data(), raw_vecs[i].data(), DIMENSION); + EXPECT_NEAR(d, expected, 1e-4) << "i=" << i; + } + + // Batch via DistanceImpl. + ASSERT_TRUE(dist_impl.batch_valid()); + std::vector dp_list(COUNT - 1); + for (size_t i = 1; i < COUNT; ++i) { + dp_list[i - 1] = quant_vecs[i].data(); + } + std::vector batch_results(COUNT - 1); + dist_impl.batch(dp_list.data(), dp_list.size(), batch_results.data()); + for (size_t i = 0; i < dp_list.size(); ++i) { + float expected = reference_cosine(raw_vecs[0].data(), + raw_vecs[i + 1].data(), DIMENSION); + EXPECT_NEAR(batch_results[i], expected, 1e-4) << "i=" << i; + } + } + + // --- calc_distance_dp_dp (pairwise) --- + for (size_t i = 1; i < 10; ++i) { + float d = quantizer->calc_distance_dp_dp(quant_vecs[i].data(), + quant_vecs[0].data()); + float expected = + reference_cosine(raw_vecs[i].data(), raw_vecs[0].data(), DIMENSION); + EXPECT_NEAR(d, expected, 1e-4) << "i=" << i; + } + + // --- calc_distance_dp_query_unquantized --- + for (size_t i = 1; i < 10; ++i) { + float d = quantizer->calc_distance_dp_query_unquantized( + quant_vecs[i].data(), raw_vecs[0].data()); + float expected = + reference_cosine(raw_vecs[i].data(), raw_vecs[0].data(), DIMENSION); + EXPECT_NEAR(d, expected, 1e-4) << "i=" << i; + } } \ No newline at end of file From 628982db5738d6e431e37d362e9f85816d93e09c Mon Sep 17 00:00:00 2001 From: ray Date: Wed, 1 Jul 2026 11:53:21 +0800 Subject: [PATCH 10/25] fix: fix cosine --- src/turbo/distance/scalar/fp32/cosine.cc | 31 +++++++------------ .../quantizer/fp32_quantizer/fp32_quantizer.h | 2 +- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/src/turbo/distance/scalar/fp32/cosine.cc b/src/turbo/distance/scalar/fp32/cosine.cc index 112a6fc80..6dd9fc532 100644 --- a/src/turbo/distance/scalar/fp32/cosine.cc +++ b/src/turbo/distance/scalar/fp32/cosine.cc @@ -13,34 +13,27 @@ // limitations under the License. #include "scalar/fp32/cosine.h" -#include +#include "scalar/fp32/inner_product.h" namespace zvec::turbo::scalar { void cosine_fp32_distance(const void *a, const void *b, size_t dim, float *distance) { - const float *fa = static_cast(a); - const float *fb = static_cast(b); - float dot = 0.0f; - float na = 0.0f; - float nb = 0.0f; - for (size_t i = 0; i < dim; ++i) { - dot += fa[i] * fb[i]; - na += fa[i] * fa[i]; - nb += fb[i] * fb[i]; - } - float denom = std::sqrt(na) * std::sqrt(nb); - if (denom < 1e-12f) { - *distance = 1.0f; - } else { - *distance = 1.0f - dot / denom; - } + // inner_product_fp32_distance returns -real_IP; cosine = 1 - real_IP = 1 + + // ip. + float ip; + inner_product_fp32_distance(a, b, dim, &ip); + + *distance = 1 + ip; } void cosine_fp32_batch_distance(const void *const *vectors, const void *query, size_t n, size_t dim, float *distances) { - for (size_t i = 0; i < n; ++i) { - cosine_fp32_distance(vectors[i], query, dim, &distances[i]); + inner_product_fp32_batch_distance(vectors, query, n, dim, distances); + // inner_product batch returns -real_IP per element; cosine = 1 - real_IP = 1 + // + d. + for (size_t i = 0; i < n; i++) { + distances[i] = 1 + distances[i]; } } diff --git a/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h b/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h index 79a7caa09..998a19d68 100644 --- a/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h +++ b/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h @@ -57,7 +57,7 @@ class Fp32Quantizer : public Quantizer { } int train(core::IndexHolder::Pointer /*holder*/) override { - return IndexError_NotImplemented; + return 0; } size_t quantized_datapoint_vector_length() const override { From de3322042a2b7f300d99b6403c47666135676f91 Mon Sep 17 00:00:00 2001 From: ray Date: Wed, 1 Jul 2026 14:17:05 +0800 Subject: [PATCH 11/25] fix: add quantizer --- src/turbo/distance/scalar/fp32/inner_product.cc | 4 ++-- src/turbo/quantizer/fp32_quantizer/fp32_quantizer.cc | 6 ------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/turbo/distance/scalar/fp32/inner_product.cc b/src/turbo/distance/scalar/fp32/inner_product.cc index 63a0575a3..1f966d788 100644 --- a/src/turbo/distance/scalar/fp32/inner_product.cc +++ b/src/turbo/distance/scalar/fp32/inner_product.cc @@ -16,8 +16,8 @@ namespace zvec::turbo::scalar { -// Compute squared Euclidean distance between a single quantized FP32 -// vector pair. +// Compute negated inner product between a single FP32 vector pair. +// Returns -dot(a, b) so that callers can derive cosine distance as 1 + ip. void inner_product_fp32_distance(const void *a, const void *b, size_t dim, float *distance) { const float *m = reinterpret_cast(a); diff --git a/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.cc b/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.cc index efd4e951e..6f8853c5c 100644 --- a/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.cc +++ b/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.cc @@ -36,12 +36,6 @@ int Fp32Quantizer::init(const IndexMeta &meta, meta_.set_extra_meta_size(extra_meta_size_); } - // `meta.dimension()` may be either the raw dim (when the caller passes a - // bare meta, e.g. unit tests) or the inflated storage dim (data + extras) - // when an upstream converter such as CosineConverter has already appended - // a norm float and set meta.extra_meta_size(). Distinguish via the input - // meta's extra_meta_size: if it is already set, the dim is inflated and - // we strip the extras to recover the raw dim; otherwise the dim is raw. if (meta.extra_meta_size() > 0) { original_dim_ = meta.dimension() - meta.extra_meta_size() / sizeof(float); } else { From 49780f897918d2006d0e9b1b1fd95eddf0801786 Mon Sep 17 00:00:00 2001 From: ray Date: Wed, 1 Jul 2026 14:59:26 +0800 Subject: [PATCH 12/25] fix: fix quantizer --- .../fp32_quantizer/fp32_quantizer.cc | 64 +++++++++++++++---- .../quantizer/fp32_quantizer/fp32_quantizer.h | 5 +- 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.cc b/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.cc index 6f8853c5c..f5c29b76b 100644 --- a/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.cc +++ b/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -30,18 +31,14 @@ int Fp32Quantizer::init(const IndexMeta &meta, meta_.set_meta(IndexMeta::DataType::DT_FP32, meta.dimension()); + + original_dim_ = meta.dimension(); auto metric_name = meta.metric_name(); if (metric_name == "Cosine") { extra_meta_size_ = EXTRA_META_SIZE_COSINE; meta_.set_extra_meta_size(extra_meta_size_); } - if (meta.extra_meta_size() > 0) { - original_dim_ = meta.dimension() - meta.extra_meta_size() / sizeof(float); - } else { - original_dim_ = meta.dimension(); - } - // Cache the distance dispatch for the new Quantizer interface. dp_query_func_ = get_distance_func(metric_from_name(metric_name), DataType::kFp32, @@ -65,9 +62,22 @@ int Fp32Quantizer::quantize(const void *query, const IndexQueryMeta &qmeta, size_t raw_dim = (original_dim_ != 0 && qmeta.dimension() >= original_dim_) ? original_dim_ : qmeta.dimension(); - size_t byte_size = raw_dim * sizeof(float); + size_t byte_size = raw_dim * sizeof(float) + extra_meta_size_; out->resize(byte_size); - std::memcpy(&(*out)[0], query, byte_size); + + if (meta_.metric_name() == "Cosine") { + // L2-normalize the vector in-place and store the norm at the end so the + // original vector can be reconstructed during dequantize. + float *buf = reinterpret_cast(&(*out)[0]); + std::memcpy(buf, query, raw_dim * sizeof(float)); + float norm = 0.0f; + ailego::Normalizer::L2(buf, raw_dim, &norm); + std::memcpy( + reinterpret_cast(&(*out)[0]) + raw_dim * sizeof(float), + &norm, extra_meta_size_); + } else { + std::memcpy(&(*out)[0], query, byte_size); + } *ometa = qmeta; ometa->set_meta(IndexMeta::DataType::DT_FP32, raw_dim, @@ -78,9 +88,26 @@ int Fp32Quantizer::quantize(const void *query, const IndexQueryMeta &qmeta, int Fp32Quantizer::dequantize(const void *in, const IndexQueryMeta &qmeta, std::string *out) const { - size_t byte_size = qmeta.dimension() * sizeof(float); - out->resize(byte_size); - std::memcpy(out->data(), in, byte_size); + size_t raw_dim = (original_dim_ != 0 && qmeta.dimension() >= original_dim_) + ? original_dim_ + : qmeta.dimension(); + size_t byte_size = raw_dim * sizeof(float); + + if (meta_.metric_name() == "Cosine") { + // Denormalize the vector using the stored norm. + out->resize(byte_size); + const float *in_buf = reinterpret_cast(in); + float *out_buf = reinterpret_cast(&(*out)[0]); + float norm = 0.0f; + std::memcpy(&norm, reinterpret_cast(in) + byte_size, + extra_meta_size_); + for (size_t i = 0; i < raw_dim; ++i) { + out_buf[i] = in_buf[i] * norm; + } + } else { + out->resize(byte_size); + std::memcpy(out->data(), in, byte_size); + } return 0; } @@ -103,8 +130,19 @@ DistanceImpl Fp32Quantizer::distance(const void *query, } void Fp32Quantizer::quantize_one(const void *input, void *output) const { - std::memcpy(output, input, - static_cast(original_dim_) * sizeof(float)); + size_t byte_size = static_cast(original_dim_) * sizeof(float); + + if (meta_.metric_name() == "Cosine") { + // L2-normalize and store the norm at the end. + std::memcpy(output, input, byte_size); + float *buf = reinterpret_cast(output); + float norm = 0.0f; + ailego::Normalizer::L2(buf, original_dim_, &norm); + std::memcpy(reinterpret_cast(output) + byte_size, &norm, + extra_meta_size_); + } else { + std::memcpy(output, input, byte_size); + } } float Fp32Quantizer::calc_distance_dp_query(const void *dp, diff --git a/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h b/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h index 998a19d68..328a71e29 100644 --- a/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h +++ b/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h @@ -102,9 +102,10 @@ class Fp32Quantizer : public Quantizer { const core::IndexQueryMeta &qmeta) const override; private: - //! Byte length of a quantized vector (raw fp32 data). + //! Byte length of a quantized vector (raw fp32 data + extra meta). size_t quantized_length() const { - return static_cast(original_dim_) * sizeof(float); + return static_cast(original_dim_) * sizeof(float) + + extra_meta_size_; } //! Quantize a single fp32 vector into a caller-provided buffer of From b1d484956fa425aaba5c3c9726dd7046e4eeb32e Mon Sep 17 00:00:00 2001 From: ray Date: Thu, 2 Jul 2026 10:58:18 +0800 Subject: [PATCH 13/25] fix: remove unused var --- src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h b/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h index 328a71e29..885390639 100644 --- a/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h +++ b/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.h @@ -116,7 +116,6 @@ class Fp32Quantizer : public Quantizer { IndexMeta meta_{}; uint32_t original_dim_{0}; - IndexMeta::DataType data_type_{}; //! Cached distance dispatch (bound in init()). DistanceFunc dp_query_func_{}; From f0fdf5d65fe8663ed89b91b43d8b25635b346e25 Mon Sep 17 00:00:00 2001 From: ray Date: Thu, 2 Jul 2026 14:08:42 +0800 Subject: [PATCH 14/25] fix: header --- examples/c++/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/c++/CMakeLists.txt b/examples/c++/CMakeLists.txt index d58814d52..4de64b45c 100644 --- a/examples/c++/CMakeLists.txt +++ b/examples/c++/CMakeLists.txt @@ -14,7 +14,10 @@ if(NOT DEFINED HOST_BUILD_DIR) endif() get_filename_component(ZVEC_ROOT_DIR "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE) -set(ZVEC_INCLUDE_DIR ${ZVEC_ROOT_DIR}/src/include) +# Include both src/include (public headers) and src (internal headers that +# are transitively reachable from public headers, e.g. +# turbo/quantizer/quantizer.h via index_factory.h). +set(ZVEC_INCLUDE_DIR ${ZVEC_ROOT_DIR}/src/include ${ZVEC_ROOT_DIR}/src) set(ZVEC_LIB_DIR ${ZVEC_ROOT_DIR}/${HOST_BUILD_DIR}/lib) # Add include and library search paths From 0c4cec659b4e285b668411ab829f1bc00e07311e Mon Sep 17 00:00:00 2001 From: ray Date: Thu, 2 Jul 2026 15:05:02 +0800 Subject: [PATCH 15/25] fix: header --- src/turbo/quantizer/quantizer.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/turbo/quantizer/quantizer.h b/src/turbo/quantizer/quantizer.h index 572d372b5..14e889787 100644 --- a/src/turbo/quantizer/quantizer.h +++ b/src/turbo/quantizer/quantizer.h @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -31,6 +30,7 @@ using namespace zvec::core; //! Magic number ('QTZR') stamped at the start of a serialized quantizer blob. constexpr uint32_t kQuantizerMagic = 0x52545A51u; + //! Current quantizer serialization format version. constexpr uint16_t kQuantizerSerVersion = 1; @@ -75,7 +75,7 @@ class Quantizer { //! Train the quantizer with a contiguous batch of data virtual int train(const void * /*data*/, size_t /*num*/, size_t /*stride*/) { - return IndexError_NotImplemented; + return 0; } //! Whether the quantizer requires training before use @@ -83,7 +83,7 @@ class Quantizer { //! Train the quantizer with data from an IndexHolder virtual int train(IndexHolder::Pointer /*holder*/) { - return IndexError_NotImplemented; + return 0; } //! Byte length of a quantized datapoint vector @@ -123,13 +123,13 @@ class Quantizer { virtual int quantize(const void * /*query*/, const IndexQueryMeta & /*qmeta*/, std::string * /*out*/, IndexQueryMeta * /*ometa*/) const { - return IndexError_NotImplemented; + return 0; } //! Dequantize a result vector back to original format virtual int dequantize(const void * /*in*/, const IndexQueryMeta & /*qmeta*/, std::string * /*out*/) const { - return IndexError_NotImplemented; + return 0; } virtual DistanceImpl distance(const void * /*query*/, @@ -139,18 +139,18 @@ class Quantizer { //! Serialize quantizer parameters virtual int serialize(std::string * /*out*/) const { - return IndexError_NotImplemented; + return 0 } //! Deserialize quantizer parameters virtual int deserialize(std::string & /*in*/) { - return IndexError_NotImplemented; + return 0; } //! Deserialize quantizer parameters from a raw, possibly mmap-backed buffer //! (zero-copy entry point for large payloads such as codebooks/matrices). virtual int deserialize(const void * /*data*/, size_t /*len*/) { - return IndexError_NotImplemented; + return 0; } protected: From 73f5bc3bab498315bac70d8c3142a7ecb530020c Mon Sep 17 00:00:00 2001 From: ray Date: Thu, 2 Jul 2026 15:36:15 +0800 Subject: [PATCH 16/25] fix: header --- src/turbo/quantizer/quantizer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/turbo/quantizer/quantizer.h b/src/turbo/quantizer/quantizer.h index 14e889787..3e5e65ec6 100644 --- a/src/turbo/quantizer/quantizer.h +++ b/src/turbo/quantizer/quantizer.h @@ -139,7 +139,7 @@ class Quantizer { //! Serialize quantizer parameters virtual int serialize(std::string * /*out*/) const { - return 0 + return 0; } //! Deserialize quantizer parameters From 437a7546c7df926ab748ce129f2cfa24083b0edc Mon Sep 17 00:00:00 2001 From: ray Date: Thu, 2 Jul 2026 16:39:13 +0800 Subject: [PATCH 17/25] fix: fix symbol --- .../quantizer/fp32_quantizer/fp32_quantizer.cc | 3 +-- src/turbo/quantizer/quantizer.h | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.cc b/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.cc index f5c29b76b..839137e6d 100644 --- a/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.cc +++ b/src/turbo/quantizer/fp32_quantizer/fp32_quantizer.cc @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include "core/quantizer/record_quantizer.h" @@ -53,7 +52,7 @@ int Fp32Quantizer::init(const IndexMeta &meta, int Fp32Quantizer::quantize(const void *query, const IndexQueryMeta &qmeta, std::string *out, IndexQueryMeta *ometa) const { if (qmeta.unit_size() != sizeof(float)) { - return IndexError_Unsupported; + return kErrUnsupported; } // qmeta.dimension() may be the inflated (data + extras) dimension when the diff --git a/src/turbo/quantizer/quantizer.h b/src/turbo/quantizer/quantizer.h index 3e5e65ec6..3b7a7d944 100644 --- a/src/turbo/quantizer/quantizer.h +++ b/src/turbo/quantizer/quantizer.h @@ -28,6 +28,20 @@ namespace turbo { using namespace zvec::core; +//! Error code literals mirroring core::IndexError::Code integer values. +//! +//! Turbo quantizer sources use these directly instead of the +//! `IndexError_NotImplemented` / `IndexError_Unsupported` const objects +//! because MSVC's WINDOWS_EXPORT_ALL_SYMBOLS does not export const data +//! with constructors from zvec_shared.dll. zvec_turbo is a static library +//! linked with /WHOLEARCHIVE, so referencing those unexported symbols across +//! the DLL boundary triggers LNK2019 on Windows. +//! +//! IndexError::Code stores -val in its constructor, so NotImplemented(11) +//! yields -11 and Unsupported(12) yields -12. +constexpr int kErrNotImplemented = -11; +constexpr int kErrUnsupported = -12; + //! Magic number ('QTZR') stamped at the start of a serialized quantizer blob. constexpr uint32_t kQuantizerMagic = 0x52545A51u; From ac17ee916c1d0851e0b18a9d541b26fbf6888f87 Mon Sep 17 00:00:00 2001 From: ray Date: Fri, 10 Jul 2026 12:45:00 +0800 Subject: [PATCH 18/25] fix: add neon --- src/include/zvec/turbo/turbo.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/include/zvec/turbo/turbo.h b/src/include/zvec/turbo/turbo.h index 2eff572d5..0b0e03c2a 100644 --- a/src/include/zvec/turbo/turbo.h +++ b/src/include/zvec/turbo/turbo.h @@ -62,12 +62,14 @@ enum class QuantizeType { enum class CpuArchType { kAuto, kScalar, + // x86 SIMD kSSE, kAVX, kAVX2, kAVX512, kAVX512VNNI, - kAVX512FP16 + kAVX512FP16, + kNEON }; DistanceFunc get_distance_func(MetricType metric_type, DataType data_type, From 7bb01c35f7c809a1ab873071beb4c151ab3c9066 Mon Sep 17 00:00:00 2001 From: ray Date: Fri, 10 Jul 2026 12:46:01 +0800 Subject: [PATCH 19/25] fix: add arm arch --- src/include/zvec/turbo/turbo.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/include/zvec/turbo/turbo.h b/src/include/zvec/turbo/turbo.h index 0b0e03c2a..e6f1587ac 100644 --- a/src/include/zvec/turbo/turbo.h +++ b/src/include/zvec/turbo/turbo.h @@ -69,7 +69,10 @@ enum class CpuArchType { kAVX512, kAVX512VNNI, kAVX512FP16, - kNEON + // ARM SIMD + kNEON, + kSVE, + kSVE2 }; DistanceFunc get_distance_func(MetricType metric_type, DataType data_type, From 403c1234b9c876d29ee5de1d142c7a49090e975c Mon Sep 17 00:00:00 2001 From: ray Date: Fri, 10 Jul 2026 18:38:23 +0800 Subject: [PATCH 20/25] fix: add meta --- src/core/framework/index_meta.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/framework/index_meta.cc b/src/core/framework/index_meta.cc index d0eadb02d..0cdf4815a 100644 --- a/src/core/framework/index_meta.cc +++ b/src/core/framework/index_meta.cc @@ -145,6 +145,7 @@ bool IndexMeta::deserialize(const void *data, size_t len) { unit_size_ = format->unit_size; element_size_ = IndexMeta::ElementSizeof(data_type_, unit_size_, dimension_); space_id_ = format->space_id; + extra_meta_size_ = format->extra_meta_size; // Read attachment ailego::Params attachment; From b3e6a56a149f3487230485180c01877b572233a2 Mon Sep 17 00:00:00 2001 From: ray Date: Fri, 10 Jul 2026 18:42:00 +0800 Subject: [PATCH 21/25] fix: query meta --- src/core/framework/index_meta.cc | 5 ++-- src/include/zvec/core/framework/index_meta.h | 25 ++++++++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/core/framework/index_meta.cc b/src/core/framework/index_meta.cc index 0cdf4815a..4a0f13e90 100644 --- a/src/core/framework/index_meta.cc +++ b/src/core/framework/index_meta.cc @@ -143,9 +143,10 @@ bool IndexMeta::deserialize(const void *data, size_t len) { data_type_ = static_cast(format->data_type); dimension_ = format->dimension; unit_size_ = format->unit_size; - element_size_ = IndexMeta::ElementSizeof(data_type_, unit_size_, dimension_); - space_id_ = format->space_id; extra_meta_size_ = format->extra_meta_size; + element_size_ = IndexMeta::ElementSizeof(data_type_, unit_size_, dimension_) + + extra_meta_size_; + space_id_ = format->space_id; // Read attachment ailego::Params attachment; diff --git a/src/include/zvec/core/framework/index_meta.h b/src/include/zvec/core/framework/index_meta.h index c6c5137db..010fca857 100644 --- a/src/include/zvec/core/framework/index_meta.h +++ b/src/include/zvec/core/framework/index_meta.h @@ -440,7 +440,8 @@ class IndexMeta { //! Set dimension of feature void set_dimension(uint32_t dim) { dimension_ = dim; - element_size_ = IndexMeta::ElementSizeof(data_type_, unit_size_, dim); + element_size_ = IndexMeta::ElementSizeof(data_type_, unit_size_, dim) + + extra_meta_size_; } //! Set meta information of feature @@ -454,7 +455,7 @@ class IndexMeta { data_type_ = data_type; dimension_ = dim; unit_size_ = unit; - element_size_ = ElementSizeof(data_type, unit, dim); + element_size_ = ElementSizeof(data_type, unit, dim) + extra_meta_size_; } //! Set meta information of feature @@ -465,6 +466,8 @@ class IndexMeta { //! Set extra meta size void set_extra_meta_size(uint32_t size) { extra_meta_size_ = size; + element_size_ = + ElementSizeof(data_type_, unit_size_, dimension_) + extra_meta_size_; } //! Set information of metric @@ -703,6 +706,16 @@ class IndexQueryMeta { return element_size_; } + //! Retrieve quantize type + uint32_t quantize_type(void) const { + return quantize_type_; + } + + //! Retrieve extra meta size in bytes + uint32_t extra_meta_size(void) const { + return extra_meta_size_; + } + //! Set dimension of feature void set_dimension(uint32_t dim) { dimension_ = dim; @@ -720,6 +733,14 @@ class IndexQueryMeta { data_type_ = data_type; } + //! Set extra meta size + void set_extra_meta_size(uint32_t size) { + extra_meta_size_ = size; + element_size_ = + IndexMeta::ElementSizeof(data_type_, unit_size_, dimension_) + + extra_meta_size_; + } + //! Set meta information of feature void set_meta(IndexMeta::DataType data_type, uint32_t unit, uint32_t dim) { data_type_ = data_type; From 428713154e48ba7326f0091c4f0221096e2eb2f9 Mon Sep 17 00:00:00 2001 From: ray Date: Wed, 15 Jul 2026 10:33:01 +0800 Subject: [PATCH 22/25] fix: fix alignment --- src/turbo/turbo.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/turbo/turbo.cc b/src/turbo/turbo.cc index 80f2ee77d..4fe8309d0 100644 --- a/src/turbo/turbo.cc +++ b/src/turbo/turbo.cc @@ -56,7 +56,9 @@ DistanceFunc get_distance_func(MetricType metric_type, DataType data_type, } } if (quantize_type == QuantizeType::kUniform) { - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) { + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI && + (cpu_arch_type == CpuArchType::kAuto || + cpu_arch_type == CpuArchType::kAVX512VNNI)) { if (metric_type == MetricType::kSquaredEuclidean) { return avx512_vnni::uniform_squared_euclidean_int8_distance; } @@ -99,7 +101,9 @@ BatchDistanceFunc get_batch_distance_func(MetricType metric_type, } } if (quantize_type == QuantizeType::kUniform) { - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) { + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI && + (cpu_arch_type == CpuArchType::kAuto || + cpu_arch_type == CpuArchType::kAVX512VNNI)) { if (metric_type == MetricType::kSquaredEuclidean) { return avx512_vnni::uniform_squared_euclidean_int8_batch_distance; } From 394356679dd61357b0376e19f54132c758692e57 Mon Sep 17 00:00:00 2001 From: ray Date: Wed, 15 Jul 2026 11:12:55 +0800 Subject: [PATCH 23/25] refactor: move header file --- examples/c++/CMakeLists.txt | 6 ++---- src/core/framework/index_factory.cc | 1 + src/include/zvec/core/framework/index_factory.h | 7 ++++++- tests/turbo/turbo_fp32_quantizer_test.cc | 1 + 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/examples/c++/CMakeLists.txt b/examples/c++/CMakeLists.txt index 4de64b45c..1e111cd48 100644 --- a/examples/c++/CMakeLists.txt +++ b/examples/c++/CMakeLists.txt @@ -14,10 +14,8 @@ if(NOT DEFINED HOST_BUILD_DIR) endif() get_filename_component(ZVEC_ROOT_DIR "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE) -# Include both src/include (public headers) and src (internal headers that -# are transitively reachable from public headers, e.g. -# turbo/quantizer/quantizer.h via index_factory.h). -set(ZVEC_INCLUDE_DIR ${ZVEC_ROOT_DIR}/src/include ${ZVEC_ROOT_DIR}/src) +# Public headers are self-contained under src/include. +set(ZVEC_INCLUDE_DIR ${ZVEC_ROOT_DIR}/src/include) set(ZVEC_LIB_DIR ${ZVEC_ROOT_DIR}/${HOST_BUILD_DIR}/lib) # Add include and library search paths diff --git a/src/core/framework/index_factory.cc b/src/core/framework/index_factory.cc index e93f57bc7..d26192eba 100644 --- a/src/core/framework/index_factory.cc +++ b/src/core/framework/index_factory.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include namespace zvec { diff --git a/src/include/zvec/core/framework/index_factory.h b/src/include/zvec/core/framework/index_factory.h index 00e77894c..1c9e287ae 100644 --- a/src/include/zvec/core/framework/index_factory.h +++ b/src/include/zvec/core/framework/index_factory.h @@ -14,7 +14,6 @@ #pragma once -#include #include #include #include @@ -30,6 +29,12 @@ #include #include +namespace zvec { +namespace turbo { +class Quantizer; +} // namespace turbo +} // namespace zvec + namespace zvec { namespace core { diff --git a/tests/turbo/turbo_fp32_quantizer_test.cc b/tests/turbo/turbo_fp32_quantizer_test.cc index c180de4a4..80610ca94 100644 --- a/tests/turbo/turbo_fp32_quantizer_test.cc +++ b/tests/turbo/turbo_fp32_quantizer_test.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include "zvec/core/framework/index_factory.h" From 084b475b4a81157bea84bb283f7e8890e80e2dd4 Mon Sep 17 00:00:00 2001 From: ray Date: Wed, 15 Jul 2026 16:44:01 +0800 Subject: [PATCH 24/25] fix: remove comment --- examples/c++/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/c++/CMakeLists.txt b/examples/c++/CMakeLists.txt index 1e111cd48..0786bdb24 100644 --- a/examples/c++/CMakeLists.txt +++ b/examples/c++/CMakeLists.txt @@ -14,7 +14,7 @@ if(NOT DEFINED HOST_BUILD_DIR) endif() get_filename_component(ZVEC_ROOT_DIR "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE) -# Public headers are self-contained under src/include. + set(ZVEC_INCLUDE_DIR ${ZVEC_ROOT_DIR}/src/include) set(ZVEC_LIB_DIR ${ZVEC_ROOT_DIR}/${HOST_BUILD_DIR}/lib) From 7552df5231c70187c28de824b3e7b272adbd8fd0 Mon Sep 17 00:00:00 2001 From: ray Date: Wed, 15 Jul 2026 17:26:57 +0800 Subject: [PATCH 25/25] fix: fix empty line --- examples/c++/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/c++/CMakeLists.txt b/examples/c++/CMakeLists.txt index 0786bdb24..f5a19f056 100644 --- a/examples/c++/CMakeLists.txt +++ b/examples/c++/CMakeLists.txt @@ -12,7 +12,6 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) if(NOT DEFINED HOST_BUILD_DIR) set(HOST_BUILD_DIR "build") endif() - get_filename_component(ZVEC_ROOT_DIR "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE) set(ZVEC_INCLUDE_DIR ${ZVEC_ROOT_DIR}/src/include)