From 9567f30051cf2db7c62520e70ade9c81fcae8a56 Mon Sep 17 00:00:00 2001 From: Qinren Zhou Date: Sun, 25 Oct 2026 20:18:18 +0800 Subject: [PATCH] feat: upgrade zvec to 0.8.0 and adapt Node.js bindings --- CMakeLists.txt | 75 ++++++++++------- package.json | 2 - packages/bindings-darwin-arm64/package.json | 4 +- packages/bindings-darwin-x64/package.json | 4 +- .../bindings-linux-arm64-musl/package.json | 4 +- packages/bindings-linux-arm64/package.json | 4 +- packages/bindings-linux-x64-musl/package.json | 4 +- packages/bindings-linux-x64/package.json | 4 +- packages/bindings-win32-arm64/package.json | 4 +- packages/bindings-win32-x64/package.json | 4 +- scripts/lib/artifacts.js | 19 ----- scripts/pack-local.js | 80 +++++++------------ src/binding/config.cc | 35 -------- src/binding/config.h | 4 - src/binding/params.cc | 41 ++++++++-- src/index.d.mts | 6 +- src/index.d.ts | 54 ++++++------- src/index.js | 17 ---- src/index.mjs | 2 - src/zvec | 2 +- tests/schema.test.ts | 11 ++- 21 files changed, 152 insertions(+), 228 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9e95204..b423d8b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,7 +37,6 @@ endif() set(ZVEC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/zvec) set(ZVEC_BINARY_DIR ${CMAKE_BINARY_DIR}/zvec-build) set(ZVEC_DEPENDENCY_LIB_DIR ${ZVEC_BINARY_DIR}/external/usr/local/lib) -set(ZVEC_JIEBA_DICT_SRC ${ZVEC_SOURCE_DIR}/thirdparty/cppjieba/cppjieba-5.6.7/dict) get_property(is_multi_config GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) if(is_multi_config) @@ -67,17 +66,40 @@ else() ) endif() -# ExternalProject preserves zvec's files, but not its CMake target graph. On -# Linux x86-64, native zvec keeps the runtime-dispatched RaBitQ implementation -# in a separate static archive and propagates it through -# rabitqlib -> rabitqlib_runtime. libzvec_core.a therefore contains callers, -# not these implementations, so forward the archive explicitly when linking -# the addon. Other platforms disable RaBitQ and do not produce this archive. -if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND - CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64|AMD64)$") - set(ZVEC_RABITQ_RUNTIME_LIB - ${ZVEC_LIB_DIR}/librabitqlib_runtime.a) - list(APPEND ZVEC_BYPRODUCTS ${ZVEC_RABITQ_RUNTIME_LIB}) +# ExternalProject does not preserve zvec's CMake target graph. Mirror its +# RaBitQ support checks and forward rabitq_fastscan -> rabitq_core explicitly. +set(ZVEC_RABITQ_SUPPORTED OFF) +set(ZVEC_RABITQ_LIBS) +if(NOT ANDROID AND NOT IOS AND CMAKE_SIZEOF_VOID_P EQUAL 8 + AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64|AMD64)$" + AND (CMAKE_SYSTEM_NAME STREQUAL "Linux" OR (WIN32 AND MSVC))) + if(MSVC) + set(ZVEC_RABITQ_AVX2_FLAGS "/arch:AVX2") + set(ZVEC_RABITQ_AVX512_FLAGS "/arch:AVX512") + set(ZVEC_RABITQ_AVX512_POPCNT_FLAGS "/arch:AVX512") + else() + set(ZVEC_RABITQ_AVX2_FLAGS "-mavx2 -mfma") + set(ZVEC_RABITQ_AVX512_FLAGS "-mavx512f -mavx512bw -mavx512dq -mfma") + set(ZVEC_RABITQ_AVX512_POPCNT_FLAGS + "-mavx512f -mavx512bw -mavx512dq -mavx512vpopcntdq -mfma") + endif() + + include(CheckCXXCompilerFlag) + set(ZVEC_RABITQ_SUPPORTED ON) + foreach(variant AVX2 AVX512 AVX512_POPCNT) + check_cxx_compiler_flag("${ZVEC_RABITQ_${variant}_FLAGS}" + ZVEC_COMPILER_SUPPORTS_RABITQ_${variant}) + if(NOT ZVEC_COMPILER_SUPPORTS_RABITQ_${variant}) + set(ZVEC_RABITQ_SUPPORTED OFF) + endif() + endforeach() +endif() +if(ZVEC_RABITQ_SUPPORTED) + foreach(component fastscan core) + list(APPEND ZVEC_RABITQ_LIBS + ${ZVEC_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}rabitq_${component}${CMAKE_STATIC_LIBRARY_SUFFIX}) + endforeach() + list(APPEND ZVEC_BYPRODUCTS ${ZVEC_RABITQ_LIBS}) endif() @@ -146,11 +168,8 @@ endif() find_package(Threads REQUIRED) set(zvec_core_deps - zvec_turbo + ${ZVEC_RABITQ_LIBS} ) -if(ZVEC_RABITQ_RUNTIME_LIB) - list(APPEND zvec_core_deps ${ZVEC_RABITQ_RUNTIME_LIB}) -endif() if (NOT WIN32) set(zvec_ailego_deps @@ -221,12 +240,14 @@ else() ) endif() -# zvec_core: links libzvec_core.a via special flags (handled externally), but declare logical deps +# Core and Turbo register factories through static initializers. Keep all their +# objects so algorithms and quantizers requested by name remain available. add_library(zvec-core INTERFACE) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") target_link_libraries(zvec-core INTERFACE -Wl,--whole-archive zvec_core + zvec_turbo -Wl,--no-whole-archive -Wl,--start-group zvec-ailego @@ -236,6 +257,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") elseif(APPLE) target_link_libraries(zvec-core INTERFACE "-Wl,-force_load,${ZVEC_LIB_DIR}/libzvec_core.a" + "-Wl,-force_load,${ZVEC_LIB_DIR}/libzvec_turbo.a" zvec-ailego ${zvec_core_deps} ) @@ -243,6 +265,7 @@ elseif(ANDROID) target_link_libraries(zvec-core INTERFACE -Wl,--whole-archive zvec_core + zvec_turbo -Wl,--no-whole-archive -Wl,--start-group zvec-ailego @@ -252,10 +275,14 @@ elseif(ANDROID) elseif(WIN32) target_link_libraries(zvec-core INTERFACE zvec_core + zvec_turbo zvec-ailego ${zvec_core_deps} ) - target_link_options(zvec-core INTERFACE "/WHOLEARCHIVE:zvec_core") + target_link_options(zvec-core INTERFACE + "/WHOLEARCHIVE:zvec_core" + "/WHOLEARCHIVE:zvec_turbo" + ) else() message(FATAL_ERROR "Unsupported platform: ${CMAKE_SYSTEM_NAME}") endif() @@ -380,18 +407,6 @@ if(UNIX) set_property(TARGET ${PROJECT_NAME} PROPERTY SKIP_BUILD_RPATH TRUE) endif() -add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - "$/jieba_dict" - COMMAND ${CMAKE_COMMAND} -E copy_if_different - "${ZVEC_JIEBA_DICT_SRC}/jieba.dict.utf8" - "$/jieba_dict/jieba.dict.utf8" - COMMAND ${CMAKE_COMMAND} -E copy_if_different - "${ZVEC_JIEBA_DICT_SRC}/hmm_model.utf8" - "$/jieba_dict/hmm_model.utf8" - COMMENT "Copying Jieba tokenizer dictionary" -) - # Strip symbols to reduce executable size if(CMAKE_BUILD_TYPE STREQUAL "Release") if(APPLE) diff --git a/package.json b/package.json index 7378430..054a95c 100644 --- a/package.json +++ b/package.json @@ -43,8 +43,6 @@ "src/index.d.mts", "scripts/install.js", "zvec_node_binding.node", - "jieba_dict/jieba.dict.utf8", - "jieba_dict/hmm_model.utf8", "README.md", "LICENSE" ], diff --git a/packages/bindings-darwin-arm64/package.json b/packages/bindings-darwin-arm64/package.json index cb10c38..e1ad6ad 100644 --- a/packages/bindings-darwin-arm64/package.json +++ b/packages/bindings-darwin-arm64/package.json @@ -10,8 +10,6 @@ ], "main": "zvec_node_binding.node", "files": [ - "zvec_node_binding.node", - "jieba_dict/jieba.dict.utf8", - "jieba_dict/hmm_model.utf8" + "zvec_node_binding.node" ] } diff --git a/packages/bindings-darwin-x64/package.json b/packages/bindings-darwin-x64/package.json index eab1b24..2727bd8 100644 --- a/packages/bindings-darwin-x64/package.json +++ b/packages/bindings-darwin-x64/package.json @@ -10,8 +10,6 @@ ], "main": "zvec_node_binding.node", "files": [ - "zvec_node_binding.node", - "jieba_dict/jieba.dict.utf8", - "jieba_dict/hmm_model.utf8" + "zvec_node_binding.node" ] } diff --git a/packages/bindings-linux-arm64-musl/package.json b/packages/bindings-linux-arm64-musl/package.json index e0d2011..0623b93 100644 --- a/packages/bindings-linux-arm64-musl/package.json +++ b/packages/bindings-linux-arm64-musl/package.json @@ -13,8 +13,6 @@ ], "main": "zvec_node_binding.node", "files": [ - "zvec_node_binding.node", - "jieba_dict/jieba.dict.utf8", - "jieba_dict/hmm_model.utf8" + "zvec_node_binding.node" ] } diff --git a/packages/bindings-linux-arm64/package.json b/packages/bindings-linux-arm64/package.json index 78700f3..565e2d5 100644 --- a/packages/bindings-linux-arm64/package.json +++ b/packages/bindings-linux-arm64/package.json @@ -13,8 +13,6 @@ ], "main": "zvec_node_binding.node", "files": [ - "zvec_node_binding.node", - "jieba_dict/jieba.dict.utf8", - "jieba_dict/hmm_model.utf8" + "zvec_node_binding.node" ] } diff --git a/packages/bindings-linux-x64-musl/package.json b/packages/bindings-linux-x64-musl/package.json index f22d714..5d67f8c 100644 --- a/packages/bindings-linux-x64-musl/package.json +++ b/packages/bindings-linux-x64-musl/package.json @@ -13,8 +13,6 @@ ], "main": "zvec_node_binding.node", "files": [ - "zvec_node_binding.node", - "jieba_dict/jieba.dict.utf8", - "jieba_dict/hmm_model.utf8" + "zvec_node_binding.node" ] } diff --git a/packages/bindings-linux-x64/package.json b/packages/bindings-linux-x64/package.json index c5ed46b..8fceeaa 100644 --- a/packages/bindings-linux-x64/package.json +++ b/packages/bindings-linux-x64/package.json @@ -13,8 +13,6 @@ ], "main": "zvec_node_binding.node", "files": [ - "zvec_node_binding.node", - "jieba_dict/jieba.dict.utf8", - "jieba_dict/hmm_model.utf8" + "zvec_node_binding.node" ] } diff --git a/packages/bindings-win32-arm64/package.json b/packages/bindings-win32-arm64/package.json index b75c119..9c944d4 100644 --- a/packages/bindings-win32-arm64/package.json +++ b/packages/bindings-win32-arm64/package.json @@ -10,8 +10,6 @@ ], "main": "zvec_node_binding.node", "files": [ - "zvec_node_binding.node", - "jieba_dict/jieba.dict.utf8", - "jieba_dict/hmm_model.utf8" + "zvec_node_binding.node" ] } diff --git a/packages/bindings-win32-x64/package.json b/packages/bindings-win32-x64/package.json index ab2ed93..fc67e9c 100644 --- a/packages/bindings-win32-x64/package.json +++ b/packages/bindings-win32-x64/package.json @@ -10,8 +10,6 @@ ], "main": "zvec_node_binding.node", "files": [ - "zvec_node_binding.node", - "jieba_dict/jieba.dict.utf8", - "jieba_dict/hmm_model.utf8" + "zvec_node_binding.node" ] } diff --git a/scripts/lib/artifacts.js b/scripts/lib/artifacts.js index 341e773..e832e0a 100644 --- a/scripts/lib/artifacts.js +++ b/scripts/lib/artifacts.js @@ -4,11 +4,6 @@ const path = require('path'); const BINARY_FILENAME = 'zvec_node_binding.node'; -const JIEBA_DICT_DIRNAME = 'jieba_dict'; -const JIEBA_DICT_FILES = [ - 'jieba.dict.utf8', - 'hmm_model.utf8', -]; function getBuildOutputDir(packageRoot, buildType = 'Release') { @@ -22,7 +17,6 @@ function getBuildArtifacts(packageRoot, buildType = 'Release') { return { outputDir, binaryPath: path.join(outputDir, BINARY_FILENAME), - jiebaDictDir: path.join(outputDir, JIEBA_DICT_DIRNAME), }; } @@ -32,12 +26,6 @@ function verifyBuildArtifacts(packageRoot, buildType = 'Release') { if (!fs.existsSync(artifacts.binaryPath)) { throw new Error(`Binary not found at ${artifacts.binaryPath}`); } - for (const filename of JIEBA_DICT_FILES) { - const filePath = path.join(artifacts.jiebaDictDir, filename); - if (!fs.existsSync(filePath)) { - throw new Error(`Jieba dictionary file not found at ${filePath}`); - } - } return artifacts; } @@ -47,17 +35,11 @@ function copyBuildArtifacts(packageRoot, targetDir, buildType = 'Release') { fs.mkdirSync(targetDir, { recursive: true }); const binaryPath = path.join(targetDir, BINARY_FILENAME); - const jiebaDictDir = path.join(targetDir, JIEBA_DICT_DIRNAME); fs.copyFileSync(artifacts.binaryPath, binaryPath); - fs.cpSync(artifacts.jiebaDictDir, jiebaDictDir, { - recursive: true, - force: true, - }); return { targetDir, binaryPath, - jiebaDictDir, }; } @@ -79,7 +61,6 @@ function stagePrebuiltArtifacts(packageRoot, prebuiltTarget, buildType = 'Releas module.exports = { BINARY_FILENAME, - JIEBA_DICT_DIRNAME, verifyBuildArtifacts, stageLocalArtifacts, stagePrebuiltArtifacts, diff --git a/scripts/pack-local.js b/scripts/pack-local.js index adda669..7a17e6e 100644 --- a/scripts/pack-local.js +++ b/scripts/pack-local.js @@ -6,67 +6,47 @@ const fs = require('fs'); const path = require('path'); const { BINARY_FILENAME, - JIEBA_DICT_DIRNAME, stageLocalArtifacts, } = require('./lib/artifacts'); const PACKAGE_ROOT = path.resolve(__dirname, '..'); -const packageJsonPath = path.join(PACKAGE_ROOT, 'package.json'); -const originalPackageJson = fs.readFileSync(packageJsonPath, 'utf8'); -const backupDir = fs.mkdtempSync(path.join(PACKAGE_ROOT, '.zvec-pack-local-')); -const bundledEntries = [BINARY_FILENAME, JIEBA_DICT_DIRNAME]; -const entriesToRestore = []; - - -function backupBundledEntries() { - for (const entry of bundledEntries) { - const sourcePath = path.join(PACKAGE_ROOT, entry); - if (fs.existsSync(sourcePath)) { - fs.renameSync(sourcePath, path.join(backupDir, entry)); - } - // Restore only entries whose original state has been safely preserved. - entriesToRestore.push(entry); - } -} - - -function restoreBundledEntries() { - for (const entry of entriesToRestore) { - const targetPath = path.join(PACKAGE_ROOT, entry); - const backupPath = path.join(backupDir, entry); - fs.rmSync(targetPath, { recursive: true, force: true }); - if (fs.existsSync(backupPath)) { - fs.renameSync(backupPath, targetPath); - } - } - fs.rmSync(backupDir, { recursive: true, force: true }); -} try { - // Build the bindings - execSync('npm run build', { stdio: 'inherit', cwd: PACKAGE_ROOT }); + execSync('npm run build:release', { stdio: 'inherit', cwd: PACKAGE_ROOT }); - // A local package bundles the just-built addon directly in the main package. - // Preserve any existing local artifacts and restore them after packing. - backupBundledEntries(); - const staged = stageLocalArtifacts(PACKAGE_ROOT); - console.log(`Binary staged for local package at ${staged.binaryPath}`); - console.log(`Jieba dictionary staged at ${staged.jiebaDictDir}`); - - // Temporarily remove optionalDependencies from package.json - // (local pack bundles the binary directly, no need for platform packages) + const packageJsonPath = path.join(PACKAGE_ROOT, 'package.json'); + const originalPackageJson = fs.readFileSync(packageJsonPath, 'utf8'); const pkg = JSON.parse(originalPackageJson); - delete pkg.optionalDependencies; - fs.writeFileSync(packageJsonPath, JSON.stringify(pkg, null, 2) + '\n'); - - // Pack - execSync('npm pack', { stdio: 'inherit', cwd: PACKAGE_ROOT, encoding: 'utf8' }); + const binaryPath = path.join(PACKAGE_ROOT, BINARY_FILENAME); + const backupDir = fs.mkdtempSync(path.join(PACKAGE_ROOT, '.zvec-pack-local-')); + const backupPath = path.join(backupDir, BINARY_FILENAME); + let binaryPrepared = false; + + try { + if (fs.existsSync(binaryPath)) { + fs.renameSync(binaryPath, backupPath); + } + binaryPrepared = true; + + const staged = stageLocalArtifacts(PACKAGE_ROOT); + console.log(`Binary staged for local package at ${staged.binaryPath}`); + + delete pkg.optionalDependencies; + fs.writeFileSync(packageJsonPath, JSON.stringify(pkg, null, 2) + '\n'); + execSync('npm pack', { stdio: 'inherit', cwd: PACKAGE_ROOT }); + } finally { + fs.writeFileSync(packageJsonPath, originalPackageJson); + if (binaryPrepared) { + fs.rmSync(binaryPath, { force: true }); + if (fs.existsSync(backupPath)) { + fs.renameSync(backupPath, binaryPath); + } + } + fs.rmSync(backupDir, { recursive: true, force: true }); + } } catch (error) { console.error('❌ Error during build and packaging:', error.message); process.exitCode = 1; -} finally { - fs.writeFileSync(packageJsonPath, originalPackageJson); - restoreBundledEntries(); } diff --git a/src/binding/config.cc b/src/binding/config.cc index 7ecf2f7..e37ed30 100644 --- a/src/binding/config.cc +++ b/src/binding/config.cc @@ -187,17 +187,6 @@ Napi::Value Initialize(const Napi::CallbackInfo &info) { } } - if (obj.Has("jiebaDictDir")) { - if (obj.Get("jiebaDictDir").IsString()) { - config.jieba_dict_dir = - obj.Get("jiebaDictDir").As().Utf8Value(); - } else { - ThrowIfNotOk(env, zvec::Status::InvalidArgument( - "Expected a string for 'jiebaDictDir'")); - return env.Undefined(); - } - } - if (auto s = zvec::GlobalConfig::Instance().initialize(config); !s.ok()) { ThrowIfNotOk(env, s); } @@ -205,26 +194,6 @@ Napi::Value Initialize(const Napi::CallbackInfo &info) { } -Napi::Value SetDefaultJiebaDictDir(const Napi::CallbackInfo &info) { - Napi::Env env = info.Env(); - if (info.Length() != 1 || !info[0].IsString()) { - ThrowIfNotOk(env, zvec::Status::InvalidArgument( - "setDefaultJiebaDictDir() expects exactly one " - "argument: (dir: string)")); - return env.Undefined(); - } - zvec::GlobalConfig::Instance().set_default_jieba_dict_dir( - info[0].As().Utf8Value()); - return env.Undefined(); -} - - -Napi::Value GetDefaultJiebaDictDir(const Napi::CallbackInfo &info) { - return Napi::String::New(info.Env(), - zvec::GlobalConfig::Instance().jieba_dict_dir()); -} - - Napi::Value GetIOBackendType(const Napi::CallbackInfo &info) { return Napi::Number::New( info.Env(), @@ -243,10 +212,6 @@ Napi::Object InitConfig(Napi::Env env, Napi::Object exports) { exports.Set("LogLevel", CreateLogLevelObject(env)); exports.Set("IOBackendType", CreateIOBackendTypeObject(env)); exports.Set("initialize", Napi::Function::New(env, Initialize)); - exports.Set("setDefaultJiebaDictDir", - Napi::Function::New(env, SetDefaultJiebaDictDir)); - exports.Set("getDefaultJiebaDictDir", - Napi::Function::New(env, GetDefaultJiebaDictDir)); exports.Set("getIOBackendType", Napi::Function::New(env, GetIOBackendType)); exports.Set("getIOBackendDescription", Napi::Function::New(env, GetIOBackendDescription)); diff --git a/src/binding/config.h b/src/binding/config.h index 0f7cd82..d1c75af 100644 --- a/src/binding/config.h +++ b/src/binding/config.h @@ -15,10 +15,6 @@ Napi::Object CreateIOBackendTypeObject(Napi::Env env); Napi::Value Initialize(const Napi::CallbackInfo &info); -Napi::Value SetDefaultJiebaDictDir(const Napi::CallbackInfo &info); - -Napi::Value GetDefaultJiebaDictDir(const Napi::CallbackInfo &info); - Napi::Value GetIOBackendType(const Napi::CallbackInfo &info); Napi::Value GetIOBackendDescription(const Napi::CallbackInfo &info); diff --git a/src/binding/params.cc b/src/binding/params.cc index a3850b1..0300d6c 100644 --- a/src/binding/params.cc +++ b/src/binding/params.cc @@ -1366,8 +1366,19 @@ zvec::Result ParseHnswQueryParams( } } - return std::make_shared(ef, radius, is_linear, - is_using_refiner); + float scale_factor{0.0f}; + if (obj.Has("scaleFactor")) { + if (!obj.Get("scaleFactor").IsNumber()) { + return tl::make_unexpected( + zvec::Status::InvalidArgument("Expected a number for 'scaleFactor'")); + } + scale_factor = obj.Get("scaleFactor").As().FloatValue(); + } + + auto params = std::make_shared(ef, radius, is_linear, + is_using_refiner); + params->set_scale_factor(scale_factor); + return params; } @@ -1413,8 +1424,17 @@ zvec::Result ParseHnswRabitqQueryParams( } } - return std::make_shared(ef, radius, is_linear, - is_using_refiner); + float scale_factor{0.0f}; + if (obj.Has("scaleFactor")) { + if (!obj.Get("scaleFactor").IsNumber()) { + return tl::make_unexpected( + zvec::Status::InvalidArgument("Expected a number for 'scaleFactor'")); + } + scale_factor = obj.Get("scaleFactor").As().FloatValue(); + } + + return std::make_shared( + ef, radius, is_linear, is_using_refiner, scale_factor); } @@ -1460,8 +1480,17 @@ zvec::Result ParseIVFQueryParams( } } - auto params = - std::make_shared(nprobe, is_using_refiner); + float scale_factor{10.0f}; + if (obj.Has("scaleFactor")) { + if (!obj.Get("scaleFactor").IsNumber()) { + return tl::make_unexpected( + zvec::Status::InvalidArgument("Expected a number for 'scaleFactor'")); + } + scale_factor = obj.Get("scaleFactor").As().FloatValue(); + } + + auto params = std::make_shared(nprobe, is_using_refiner, + scale_factor); params->set_radius(radius); params->set_is_linear(is_linear); return params; diff --git a/src/index.d.mts b/src/index.d.mts index bbcdf26..ece635c 100644 --- a/src/index.d.mts +++ b/src/index.d.mts @@ -3,15 +3,13 @@ export { ZVecCollectionSchema, ZVecCreateAndOpen, ZVecDataType, - ZVecGetDefaultJiebaDictDir, ZVecGetIOBackendDescription, ZVecGetIOBackendType, ZVecIndexType, ZVecInitialize, ZVecIOBackendType, ZVecLogLevel, ZVecLogType, ZVecMetricType, ZVecOpen, - ZVecQuantizeType, - ZVecSetDefaultJiebaDictDir + ZVecQuantizeType } from './index.js'; export type { @@ -62,8 +60,6 @@ declare const _default: { ZVecIOBackendType: typeof import('./index.js').ZVecIOBackendType; ZVecCollectionSchema: typeof import('./index.js').ZVecCollectionSchema; ZVecInitialize: typeof import('./index.js').ZVecInitialize; - ZVecSetDefaultJiebaDictDir: typeof import('./index.js').ZVecSetDefaultJiebaDictDir; - ZVecGetDefaultJiebaDictDir: typeof import('./index.js').ZVecGetDefaultJiebaDictDir; ZVecGetIOBackendType: typeof import('./index.js').ZVecGetIOBackendType; ZVecGetIOBackendDescription: typeof import('./index.js').ZVecGetIOBackendDescription; ZVecCreateAndOpen: typeof import('./index.js').ZVecCreateAndOpen; diff --git a/src/index.d.ts b/src/index.d.ts index 400580d..7a8678d 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -221,12 +221,6 @@ export interface ZVecInitOptions { * @default undefined */ readonly ftsBruteForceByKeysRatio?: number; - - /** - * Directory containing jieba tokenizer dictionary files. - * @default undefined - */ - readonly jiebaDictDir?: string; } @@ -239,20 +233,6 @@ export interface ZVecInitOptions { */ export function ZVecInitialize(options: ZVecInitOptions): void; -/** - * Registers the process-wide default jieba dictionary directory. - * - * @group Global Configuration - */ -export function ZVecSetDefaultJiebaDictDir(dir: string): void; - -/** - * Reads the process-wide default jieba dictionary directory. - * - * @group Global Configuration - */ -export function ZVecGetDefaultJiebaDictDir(): string; - /** * Returns the active I/O backend used for DiskANN disk reads. * @@ -368,8 +348,6 @@ export interface ZVecHnswIndexParams extends ZVecIndexParams { * HNSW RaBitQ combines HNSW graph-based search with RaBitQ quantization * for improved memory efficiency while maintaining high recall. * - * **Platform support:** Only available on Linux x86_64 with AVX2/AVX-512. - * * @group Index Parameters */ export interface ZVecHnswRabitqIndexParams extends ZVecIndexParams { @@ -455,8 +433,6 @@ export interface ZVecIVFIndexParams extends ZVecIndexParams { /** * Configuration parameters for an IVF-RaBitQ index on a vector field. * - * **Platform support:** Only available on Linux x86_64 with AVX2/AVX-512. - * * @group Index Parameters */ export interface ZVecIvfRabitqIndexParams extends ZVecIndexParams { @@ -640,14 +616,20 @@ export interface ZVecHnswQueryParams extends ZVecVectorQueryParams { * @default 300 */ readonly ef?: number; + + /** + * Candidate multiplier used when isUsingRefiner is true; ignored otherwise. + * Must be finite and nonnegative. Zero uses max(topk, ef) candidates; + * positive values use floor(topk * scaleFactor), with a minimum of topk. + * @default 0 + */ + readonly scaleFactor?: number; } /** * Query-time parameters for searches performed against an HNSW-RaBitQ index. * - * **Platform support:** Only available on Linux x86_64 with AVX2/AVX-512. - * * @group Query Parameters */ export interface ZVecHnswRabitqQueryParams extends ZVecVectorQueryParams { @@ -659,6 +641,14 @@ export interface ZVecHnswRabitqQueryParams extends ZVecVectorQueryParams { * @default 300 */ readonly ef?: number; + + /** + * Candidate multiplier used when isUsingRefiner is true; ignored otherwise. + * Must be finite and nonnegative. Zero uses max(topk, ef) candidates; + * positive values use floor(topk * scaleFactor), with a minimum of topk. + * @default 0 + */ + readonly scaleFactor?: number; } @@ -676,14 +666,20 @@ export interface ZVecIVFQueryParams extends ZVecVectorQueryParams { * @default 10 */ readonly nprobe?: number; + + /** + * Candidate multiplier used when isUsingRefiner is true; ignored otherwise. + * Must be finite and nonnegative. Uses floor(topk * scaleFactor), with + * a minimum of topk candidates (including when zero). + * @default 10 + */ + readonly scaleFactor?: number; } /** * Query-time parameters for searches performed against an IVF-RaBitQ index. * - * **Platform support:** Only available on Linux x86_64 with AVX2/AVX-512. - * * @group Query Parameters */ export interface ZVecIvfRabitqQueryParams extends ZVecVectorQueryParams { @@ -1659,8 +1655,6 @@ declare const _default: { ZVecIOBackendType: typeof ZVecIOBackendType; ZVecCollectionSchema: typeof ZVecCollectionSchema; ZVecInitialize: typeof ZVecInitialize; - ZVecSetDefaultJiebaDictDir: typeof ZVecSetDefaultJiebaDictDir; - ZVecGetDefaultJiebaDictDir: typeof ZVecGetDefaultJiebaDictDir; ZVecGetIOBackendType: typeof ZVecGetIOBackendType; ZVecGetIOBackendDescription: typeof ZVecGetIOBackendDescription; ZVecCreateAndOpen: typeof ZVecCreateAndOpen; diff --git a/src/index.js b/src/index.js index e207e2c..1c0caf8 100644 --- a/src/index.js +++ b/src/index.js @@ -4,7 +4,6 @@ const { resolvePrebuiltTarget } = require('./prebuilt'); let binding; -let bindingAssetDir; let prebuiltTarget = null; @@ -12,12 +11,10 @@ try { const bundledBinaryPath = path.join(__dirname, '..', 'zvec_node_binding.node'); if (fs.existsSync(bundledBinaryPath)) { // Try bundled binary first binding = require(bundledBinaryPath); - bindingAssetDir = path.dirname(bundledBinaryPath); } else { // Fall back to platform-specific prebuilt binary package prebuiltTarget = resolvePrebuiltTarget(); const platformPackagePath = require.resolve(prebuiltTarget.packageName); binding = require(platformPackagePath); - bindingAssetDir = path.dirname(platformPackagePath); } } catch (err) { const target = prebuiltTarget?.target ?? `${process.platform}-${process.arch}`; @@ -29,18 +26,6 @@ try { } -// Register the package-bundled jieba dictionary when it sits beside the -// loaded native addon. Users can still override it explicitly afterward. -const bundledJiebaDictDir = path.join(bindingAssetDir, 'jieba_dict'); -if ( - typeof binding.setDefaultJiebaDictDir === 'function' && - fs.existsSync(path.join(bundledJiebaDictDir, 'jieba.dict.utf8')) && - fs.existsSync(path.join(bundledJiebaDictDir, 'hmm_model.utf8')) -) { - binding.setDefaultJiebaDictDir(bundledJiebaDictDir); -} - - function isZVecError(error) { return ( typeof error === "object" && @@ -62,8 +47,6 @@ module.exports = { ZVecIOBackendType: binding.IOBackendType, ZVecCollectionSchema: binding.CollectionSchema, ZVecInitialize: binding.initialize, - ZVecSetDefaultJiebaDictDir: binding.setDefaultJiebaDictDir, - ZVecGetDefaultJiebaDictDir: binding.getDefaultJiebaDictDir, ZVecGetIOBackendType: binding.getIOBackendType, ZVecGetIOBackendDescription: binding.getIOBackendDescription, ZVecCreateAndOpen: binding.createAndOpen, diff --git a/src/index.mjs b/src/index.mjs index b33d55a..177013b 100644 --- a/src/index.mjs +++ b/src/index.mjs @@ -12,8 +12,6 @@ export const ZVecLogLevel = cjs.ZVecLogLevel; export const ZVecIOBackendType = cjs.ZVecIOBackendType; export const ZVecCollectionSchema = cjs.ZVecCollectionSchema; export const ZVecInitialize = cjs.ZVecInitialize; -export const ZVecSetDefaultJiebaDictDir = cjs.ZVecSetDefaultJiebaDictDir; -export const ZVecGetDefaultJiebaDictDir = cjs.ZVecGetDefaultJiebaDictDir; export const ZVecGetIOBackendType = cjs.ZVecGetIOBackendType; export const ZVecGetIOBackendDescription = cjs.ZVecGetIOBackendDescription; export const ZVecCreateAndOpen = cjs.ZVecCreateAndOpen; diff --git a/src/zvec b/src/zvec index 58b9562..7b75327 160000 --- a/src/zvec +++ b/src/zvec @@ -1 +1 @@ -Subproject commit 58b95626f9c30e52b8fd9041dda1bcc63543041e +Subproject commit 7b7532784a781e20541b4e315919e6abc6ff0ecb diff --git a/tests/schema.test.ts b/tests/schema.test.ts index 4c6bc85..f4fd092 100644 --- a/tests/schema.test.ts +++ b/tests/schema.test.ts @@ -195,9 +195,14 @@ describe('CollectionSchema', () => { }); - const isLinuxX64 = process.platform === 'linux' && process.arch === 'x64'; + const supportsRabitq = process.arch === 'x64' && + (process.platform === 'linux' || process.platform === 'win32'); + const supportsDiskAnn = + (process.platform === 'linux' && (process.arch === 'x64' || process.arch === 'arm64')) || + (process.platform === 'darwin' && process.arch === 'arm64') || + (process.platform === 'win32' && process.arch === 'x64'); - (isLinuxX64 ? it : it.skip)('should parse HNSW-RaBitQ index params correctly', () => { + (supportsRabitq ? it : it.skip)('should parse HNSW-RaBitQ index params correctly', () => { const schema = new ZVecCollectionSchema({ name: 'test_hnsw_rabitq', vectors: [ @@ -255,7 +260,7 @@ describe('CollectionSchema', () => { }); - (isLinuxX64 ? it : it.skip)('should parse DiskANN index params correctly', () => { + (supportsDiskAnn ? it : it.skip)('should parse DiskANN index params correctly', () => { const schema = new ZVecCollectionSchema({ name: 'test_diskann', vectors: [