feat: upgrade zvec to 0.8.0 and adapt Node.js bindings

CMakeLists.txt

@@ -37,7 +37,6 @@ endif()
 set(ZVEC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/zvec)
 set(ZVEC_BINARY_DIR ${CMAKE_BINARY_DIR}/zvec-build)
 set(ZVEC_DEPENDENCY_LIB_DIR ${ZVEC_BINARY_DIR}/external/usr/local/lib)
-set(ZVEC_JIEBA_DICT_SRC ${ZVEC_SOURCE_DIR}/thirdparty/cppjieba/cppjieba-5.6.7/dict)
 
 get_property(is_multi_config GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
 if(is_multi_config)
@@ -67,17 +66,40 @@ else()
     )
 endif()
 
-# ExternalProject preserves zvec's files, but not its CMake target graph. On
-# Linux x86-64, native zvec keeps the runtime-dispatched RaBitQ implementation
-# in a separate static archive and propagates it through
-# rabitqlib -> rabitqlib_runtime. libzvec_core.a therefore contains callers,
-# not these implementations, so forward the archive explicitly when linking
-# the addon. Other platforms disable RaBitQ and do not produce this archive.
-if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND
-    CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64|AMD64)$")
-    set(ZVEC_RABITQ_RUNTIME_LIB
-        ${ZVEC_LIB_DIR}/librabitqlib_runtime.a)
-    list(APPEND ZVEC_BYPRODUCTS ${ZVEC_RABITQ_RUNTIME_LIB})
+# ExternalProject does not preserve zvec's CMake target graph. Mirror its
+# RaBitQ support checks and forward rabitq_fastscan -> rabitq_core explicitly.
+set(ZVEC_RABITQ_SUPPORTED OFF)
+set(ZVEC_RABITQ_LIBS)
+if(NOT ANDROID AND NOT IOS AND CMAKE_SIZEOF_VOID_P EQUAL 8
+    AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64|AMD64)$"
+    AND (CMAKE_SYSTEM_NAME STREQUAL "Linux" OR (WIN32 AND MSVC)))
+    if(MSVC)
+        set(ZVEC_RABITQ_AVX2_FLAGS "/arch:AVX2")
+        set(ZVEC_RABITQ_AVX512_FLAGS "/arch:AVX512")
+        set(ZVEC_RABITQ_AVX512_POPCNT_FLAGS "/arch:AVX512")
+    else()
+        set(ZVEC_RABITQ_AVX2_FLAGS "-mavx2 -mfma")
+        set(ZVEC_RABITQ_AVX512_FLAGS "-mavx512f -mavx512bw -mavx512dq -mfma")
+        set(ZVEC_RABITQ_AVX512_POPCNT_FLAGS
+            "-mavx512f -mavx512bw -mavx512dq -mavx512vpopcntdq -mfma")
+    endif()
+
+    include(CheckCXXCompilerFlag)
+    set(ZVEC_RABITQ_SUPPORTED ON)
+    foreach(variant AVX2 AVX512 AVX512_POPCNT)
+        check_cxx_compiler_flag("${ZVEC_RABITQ_${variant}_FLAGS}"
+            ZVEC_COMPILER_SUPPORTS_RABITQ_${variant})
+        if(NOT ZVEC_COMPILER_SUPPORTS_RABITQ_${variant})
+            set(ZVEC_RABITQ_SUPPORTED OFF)
+        endif()
+    endforeach()
+endif()
+if(ZVEC_RABITQ_SUPPORTED)
+    foreach(component fastscan core)
+        list(APPEND ZVEC_RABITQ_LIBS
+            ${ZVEC_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}rabitq_${component}${CMAKE_STATIC_LIBRARY_SUFFIX})
+    endforeach()
+    list(APPEND ZVEC_BYPRODUCTS ${ZVEC_RABITQ_LIBS})
 endif()
 
 
@@ -146,11 +168,8 @@ endif()
 find_package(Threads REQUIRED)
 
 set(zvec_core_deps
-    zvec_turbo
+    ${ZVEC_RABITQ_LIBS}
 )
-if(ZVEC_RABITQ_RUNTIME_LIB)
-    list(APPEND zvec_core_deps ${ZVEC_RABITQ_RUNTIME_LIB})
-endif()
 
 if (NOT WIN32)
     set(zvec_ailego_deps
@@ -221,12 +240,14 @@ else()
     )
 endif()
 
-# zvec_core: links libzvec_core.a via special flags (handled externally), but declare logical deps
+# Core and Turbo register factories through static initializers. Keep all their
+# objects so algorithms and quantizers requested by name remain available.
 add_library(zvec-core INTERFACE)
 if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
     target_link_libraries(zvec-core INTERFACE
         -Wl,--whole-archive
         zvec_core
+        zvec_turbo
         -Wl,--no-whole-archive
         -Wl,--start-group
         zvec-ailego
@@ -236,13 +257,15 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
 elseif(APPLE)
     target_link_libraries(zvec-core INTERFACE
         "-Wl,-force_load,${ZVEC_LIB_DIR}/libzvec_core.a"
+        "-Wl,-force_load,${ZVEC_LIB_DIR}/libzvec_turbo.a"
         zvec-ailego
         ${zvec_core_deps}
     )
 elseif(ANDROID)
     target_link_libraries(zvec-core INTERFACE
         -Wl,--whole-archive
         zvec_core
+        zvec_turbo
         -Wl,--no-whole-archive
         -Wl,--start-group
         zvec-ailego
@@ -252,10 +275,14 @@ elseif(ANDROID)
 elseif(WIN32)
     target_link_libraries(zvec-core INTERFACE
         zvec_core
+        zvec_turbo
         zvec-ailego
         ${zvec_core_deps}
     )
-    target_link_options(zvec-core INTERFACE "/WHOLEARCHIVE:zvec_core")
+    target_link_options(zvec-core INTERFACE
+        "/WHOLEARCHIVE:zvec_core"
+        "/WHOLEARCHIVE:zvec_turbo"
+    )
 else()
     message(FATAL_ERROR "Unsupported platform: ${CMAKE_SYSTEM_NAME}")
 endif()
@@ -380,18 +407,6 @@ if(UNIX)
     set_property(TARGET ${PROJECT_NAME} PROPERTY SKIP_BUILD_RPATH TRUE)
 endif()
 
-add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD
-    COMMAND ${CMAKE_COMMAND} -E make_directory
-        "$<TARGET_FILE_DIR:${PROJECT_NAME}>/jieba_dict"
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different
-        "${ZVEC_JIEBA_DICT_SRC}/jieba.dict.utf8"
-        "$<TARGET_FILE_DIR:${PROJECT_NAME}>/jieba_dict/jieba.dict.utf8"
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different
-        "${ZVEC_JIEBA_DICT_SRC}/hmm_model.utf8"
-        "$<TARGET_FILE_DIR:${PROJECT_NAME}>/jieba_dict/hmm_model.utf8"
-    COMMENT "Copying Jieba tokenizer dictionary"
-)
-
 # Strip symbols to reduce executable size
 if(CMAKE_BUILD_TYPE STREQUAL "Release")
     if(APPLE)

package.json

@@ -43,8 +43,6 @@
     "src/index.d.mts",
     "scripts/install.js",
     "zvec_node_binding.node",
-    "jieba_dict/jieba.dict.utf8",
-    "jieba_dict/hmm_model.utf8",
     "README.md",
     "LICENSE"
   ],

packages/bindings-darwin-arm64/package.json

@@ -10,8 +10,6 @@
   ],
   "main": "zvec_node_binding.node",
   "files": [
-    "zvec_node_binding.node",
-    "jieba_dict/jieba.dict.utf8",
-    "jieba_dict/hmm_model.utf8"
+    "zvec_node_binding.node"
   ]
 }

packages/bindings-darwin-x64/package.json

@@ -10,8 +10,6 @@
   ],
   "main": "zvec_node_binding.node",
   "files": [
-    "zvec_node_binding.node",
-    "jieba_dict/jieba.dict.utf8",
-    "jieba_dict/hmm_model.utf8"
+    "zvec_node_binding.node"
   ]
 }

packages/bindings-linux-arm64-musl/package.json

@@ -13,8 +13,6 @@
   ],
   "main": "zvec_node_binding.node",
   "files": [
-    "zvec_node_binding.node",
-    "jieba_dict/jieba.dict.utf8",
-    "jieba_dict/hmm_model.utf8"
+    "zvec_node_binding.node"
   ]
 }

packages/bindings-linux-arm64/package.json

@@ -13,8 +13,6 @@
   ],
   "main": "zvec_node_binding.node",
   "files": [
-    "zvec_node_binding.node",
-    "jieba_dict/jieba.dict.utf8",
-    "jieba_dict/hmm_model.utf8"
+    "zvec_node_binding.node"
   ]
 }

packages/bindings-linux-x64-musl/package.json

@@ -13,8 +13,6 @@
   ],
   "main": "zvec_node_binding.node",
   "files": [
-    "zvec_node_binding.node",
-    "jieba_dict/jieba.dict.utf8",
-    "jieba_dict/hmm_model.utf8"
+    "zvec_node_binding.node"
   ]
 }

packages/bindings-linux-x64/package.json

@@ -13,8 +13,6 @@
   ],
   "main": "zvec_node_binding.node",
   "files": [
-    "zvec_node_binding.node",
-    "jieba_dict/jieba.dict.utf8",
-    "jieba_dict/hmm_model.utf8"
+    "zvec_node_binding.node"
   ]
 }

packages/bindings-win32-arm64/package.json

@@ -10,8 +10,6 @@
   ],
   "main": "zvec_node_binding.node",
   "files": [
-    "zvec_node_binding.node",
-    "jieba_dict/jieba.dict.utf8",
-    "jieba_dict/hmm_model.utf8"
+    "zvec_node_binding.node"
   ]
 }

packages/bindings-win32-x64/package.json

@@ -10,8 +10,6 @@
   ],
   "main": "zvec_node_binding.node",
   "files": [
-    "zvec_node_binding.node",
-    "jieba_dict/jieba.dict.utf8",
-    "jieba_dict/hmm_model.utf8"
+    "zvec_node_binding.node"
   ]
 }

scripts/lib/artifacts.js

@@ -4,11 +4,6 @@ const path = require('path');
 
 
 const BINARY_FILENAME = 'zvec_node_binding.node';
-const JIEBA_DICT_DIRNAME = 'jieba_dict';
-const JIEBA_DICT_FILES = [
-  'jieba.dict.utf8',
-  'hmm_model.utf8',
-];
 
 
 function getBuildOutputDir(packageRoot, buildType = 'Release') {
@@ -22,7 +17,6 @@ function getBuildArtifacts(packageRoot, buildType = 'Release') {
   return {
     outputDir,
     binaryPath: path.join(outputDir, BINARY_FILENAME),
-    jiebaDictDir: path.join(outputDir, JIEBA_DICT_DIRNAME),
   };
 }
 
@@ -32,12 +26,6 @@ function verifyBuildArtifacts(packageRoot, buildType = 'Release') {
   if (!fs.existsSync(artifacts.binaryPath)) {
     throw new Error(`Binary not found at ${artifacts.binaryPath}`);
   }
-  for (const filename of JIEBA_DICT_FILES) {
-    const filePath = path.join(artifacts.jiebaDictDir, filename);
-    if (!fs.existsSync(filePath)) {
-      throw new Error(`Jieba dictionary file not found at ${filePath}`);
-    }
-  }
   return artifacts;
 }
 
@@ -47,17 +35,11 @@ function copyBuildArtifacts(packageRoot, targetDir, buildType = 'Release') {
   fs.mkdirSync(targetDir, { recursive: true });
 
   const binaryPath = path.join(targetDir, BINARY_FILENAME);
-  const jiebaDictDir = path.join(targetDir, JIEBA_DICT_DIRNAME);
   fs.copyFileSync(artifacts.binaryPath, binaryPath);
-  fs.cpSync(artifacts.jiebaDictDir, jiebaDictDir, {
-    recursive: true,
-    force: true,
-  });
 
   return {
     targetDir,
     binaryPath,
-    jiebaDictDir,
   };
 }
 
@@ -79,7 +61,6 @@ function stagePrebuiltArtifacts(packageRoot, prebuiltTarget, buildType = 'Releas
 
 module.exports = {
   BINARY_FILENAME,
-  JIEBA_DICT_DIRNAME,
   verifyBuildArtifacts,
   stageLocalArtifacts,
   stagePrebuiltArtifacts,

scripts/pack-local.js

@@ -6,67 +6,47 @@ const fs = require('fs');
 const path = require('path');
 const {
   BINARY_FILENAME,
-  JIEBA_DICT_DIRNAME,
   stageLocalArtifacts,
 } = require('./lib/artifacts');
 
 
 const PACKAGE_ROOT = path.resolve(__dirname, '..');
-const packageJsonPath = path.join(PACKAGE_ROOT, 'package.json');
-const originalPackageJson = fs.readFileSync(packageJsonPath, 'utf8');
-const backupDir = fs.mkdtempSync(path.join(PACKAGE_ROOT, '.zvec-pack-local-'));
-const bundledEntries = [BINARY_FILENAME, JIEBA_DICT_DIRNAME];
-const entriesToRestore = [];
-
-
-function backupBundledEntries() {
-  for (const entry of bundledEntries) {
-    const sourcePath = path.join(PACKAGE_ROOT, entry);
-    if (fs.existsSync(sourcePath)) {
-      fs.renameSync(sourcePath, path.join(backupDir, entry));
-    }
-    // Restore only entries whose original state has been safely preserved.
-    entriesToRestore.push(entry);
-  }
-}
-
-
-function restoreBundledEntries() {
-  for (const entry of entriesToRestore) {
-    const targetPath = path.join(PACKAGE_ROOT, entry);
-    const backupPath = path.join(backupDir, entry);
-    fs.rmSync(targetPath, { recursive: true, force: true });
-    if (fs.existsSync(backupPath)) {
-      fs.renameSync(backupPath, targetPath);
-    }
-  }
-  fs.rmSync(backupDir, { recursive: true, force: true });
-}
 
 
 try {
-  // Build the bindings
-  execSync('npm run build', { stdio: 'inherit', cwd: PACKAGE_ROOT });
+  execSync('npm run build:release', { stdio: 'inherit', cwd: PACKAGE_ROOT });
 
-  // A local package bundles the just-built addon directly in the main package.
-  // Preserve any existing local artifacts and restore them after packing.
-  backupBundledEntries();
-  const staged = stageLocalArtifacts(PACKAGE_ROOT);
-  console.log(`Binary staged for local package at ${staged.binaryPath}`);
-  console.log(`Jieba dictionary staged at ${staged.jiebaDictDir}`);
-
-  // Temporarily remove optionalDependencies from package.json
-  // (local pack bundles the binary directly, no need for platform packages)
+  const packageJsonPath = path.join(PACKAGE_ROOT, 'package.json');
+  const originalPackageJson = fs.readFileSync(packageJsonPath, 'utf8');
   const pkg = JSON.parse(originalPackageJson);
-  delete pkg.optionalDependencies;
-  fs.writeFileSync(packageJsonPath, JSON.stringify(pkg, null, 2) + '\n');
-
-  // Pack
-  execSync('npm pack', { stdio: 'inherit', cwd: PACKAGE_ROOT, encoding: 'utf8' });
+  const binaryPath = path.join(PACKAGE_ROOT, BINARY_FILENAME);
+  const backupDir = fs.mkdtempSync(path.join(PACKAGE_ROOT, '.zvec-pack-local-'));
+  const backupPath = path.join(backupDir, BINARY_FILENAME);
+  let binaryPrepared = false;
+
+  try {
+    if (fs.existsSync(binaryPath)) {
+      fs.renameSync(binaryPath, backupPath);
+    }
+    binaryPrepared = true;
+
+    const staged = stageLocalArtifacts(PACKAGE_ROOT);
+    console.log(`Binary staged for local package at ${staged.binaryPath}`);
+
+    delete pkg.optionalDependencies;
+    fs.writeFileSync(packageJsonPath, JSON.stringify(pkg, null, 2) + '\n');
+    execSync('npm pack', { stdio: 'inherit', cwd: PACKAGE_ROOT });
+  } finally {
+    fs.writeFileSync(packageJsonPath, originalPackageJson);
+    if (binaryPrepared) {
+      fs.rmSync(binaryPath, { force: true });
+      if (fs.existsSync(backupPath)) {
+        fs.renameSync(backupPath, binaryPath);
+      }
+    }
+    fs.rmSync(backupDir, { recursive: true, force: true });
+  }
 } catch (error) {
   console.error('❌ Error during build and packaging:', error.message);
   process.exitCode = 1;
-} finally {
-  fs.writeFileSync(packageJsonPath, originalPackageJson);
-  restoreBundledEntries();
 }