chore(vendor): track zvec c133f20; feat(fts)!: adapt to the embedded jieba dictionary

.github/workflows/build-prebuilt.yml

@@ -250,17 +250,9 @@ jobs:
           cp "$ZVEC_LIB_DIR/${{ matrix.lib_name }}" dist/
           # Also copy any symlinks or versioned libraries
           find "$ZVEC_LIB_DIR" -name 'libzvec_c_api*' -exec cp -P {} dist/ \; 2>/dev/null || true
-          # Bundle cppjieba's dictionary files so the `jieba` FTS tokenizer
-          # works out of the box (same layout as the upstream zvec SDK:
-          # data/jieba_dict next to the library).
-          mkdir -p dist/data/jieba_dict
-          cp vendor/zvec/thirdparty/cppjieba/cppjieba-5.6.7/dict/jieba.dict.utf8 \
-             vendor/zvec/thirdparty/cppjieba/cppjieba-5.6.7/dict/hmm_model.utf8 \
-             dist/data/jieba_dict/
           echo "${{ matrix.target }}" > dist/TARGET
           tar czf zvec-prebuilt-${{ matrix.target }}.tar.gz -C dist .
           ls -lh zvec-prebuilt-${{ matrix.target }}.tar.gz
-          ls -l dist/data/jieba_dict
         shell: bash
 
       - name: Package library (Windows)
@@ -272,17 +264,9 @@ jobs:
           find vendor/zvec/build -name 'zvec_c_api.lib' -exec cp {} dist/ \; 2>/dev/null || true
           find vendor/zvec/build -name 'zvec_c_api.dll' -path '*/Release/*' -exec cp {} dist/ \; 2>/dev/null || true
           find vendor/zvec/build -name 'zvec_c_api.lib' -path '*/Release/*' -exec cp {} dist/ \; 2>/dev/null || true
-          # Bundle cppjieba's dictionary files so the `jieba` FTS tokenizer
-          # works out of the box (same layout as the upstream zvec SDK:
-          # data/jieba_dict next to the library).
-          mkdir -p dist/data/jieba_dict
-          cp vendor/zvec/thirdparty/cppjieba/cppjieba-5.6.7/dict/jieba.dict.utf8 \
-             vendor/zvec/thirdparty/cppjieba/cppjieba-5.6.7/dict/hmm_model.utf8 \
-             dist/data/jieba_dict/
           echo "${{ matrix.target }}" > dist/TARGET
           tar czf zvec-prebuilt-${{ matrix.target }}.tar.gz -C dist .
           ls -lh zvec-prebuilt-${{ matrix.target }}.tar.gz
-          ls -l dist/data/jieba_dict
         shell: bash
 
       - name: Upload artifact

README.md

@@ -16,7 +16,7 @@ Safe, idiomatic Rust bindings for the [zvec](https://github.com/alibaba/zvec) ve
 - **Comprehensive Error Handling** — All FFI calls return `Result<T>` with detailed error codes
 - **Zero-Copy Where Possible** — Minimizes data copying across the FFI boundary
 - **Prebuilt Libraries** — Automatically downloads prebuilt `libzvec_c_api` from GitHub Releases; advanced users can override with `ZVEC_LIB_DIR`
-- **FTS with jieba Out of the Box** — Prebuilt packages ship the cppjieba dictionary (`data/jieba_dict`); `initialize` auto-discovers it and registers the default jieba dict dir for the `jieba` FTS tokenizer
+- **FTS with jieba Out of the Box** — `libzvec_c_api` embeds cppjieba's default dictionary and HMM model (LZ4-compressed), so the `jieba` FTS tokenizer needs no dictionary files on disk and no registration step
 
 ## Supported Platforms
 
@@ -250,8 +250,6 @@ cargo run --example vector_search
 | `shutdown()` | Release all resources |
 | `version()` | Get version string |
 | `is_initialized()` | Check initialization status |
-| `set_default_jieba_dict_dir(dir)` | Set the process-wide default jieba dict dir for the FTS `jieba` tokenizer |
-| `get_default_jieba_dict_dir()` | Get the current default jieba dict dir (`""` when unset) |
 | `io_backend_type()` / `io_backend_type_name(code)` | Active DiskAnn I/O backend code and name (`io_uring`, `libaio`, `pread`, `windows_overlapped`) |
 | `io_backend_description()` | Human-readable description of the active I/O backend |
 
@@ -268,14 +266,13 @@ initialize(Some(&config))?;
 
 #### jieba FTS tokenizer dictionary
 
-The `jieba` FTS tokenizer needs cppjieba's dictionary files (`jieba.dict.utf8`
-and `hmm_model.utf8`). The prebuilt libraries ship them under `data/jieba_dict`
-next to `libzvec_c_api`, and `initialize` automatically discovers that
-directory and registers it via `zvec_set_default_jieba_dict_dir` — Chinese
-full-text search works with no extra configuration:
+`libzvec_c_api` embeds cppjieba's default dictionary and HMM model
+(LZ4-compressed), so the `jieba` FTS tokenizer reads them from the library
+itself — Chinese full-text search works with no dictionary files on disk and
+no extra configuration:
 
 ```rust
-initialize(None)?; // jieba dict auto-discovered and registered
+initialize(None)?;
 
 let schema = CollectionSchema::builder("articles")
     .add_field(FieldSchema::new("id", DataType::String, false, 0)?)
@@ -284,12 +281,15 @@ let schema = CollectionSchema::builder("articles")
     .build()?;
 ```
 
-To override the location (e.g. a custom dictionary), use one of:
+To use a custom dictionary instead of the embedded default, set one of the
+following (highest priority first); the directory must contain
+`jieba.dict.utf8` and `hmm_model.utf8`:
 
-- `ConfigBuilder::new().jieba_dict_dir("/path/to/jieba_dict")` — per `initialize` call (highest priority)
-- `set_default_jieba_dict_dir("/path/to/jieba_dict")` — process-wide default
-- `ZVEC_JIEBA_DICT_DIR` environment variable — read by the zvec library at query time
 - per-field `extra_params` JSON with `"jieba_dict_dir"` — per index
+- `ZVEC_JIEBA_DICT_DIR` environment variable — process-wide, read when the tokenizer initializes
+
+The `user_dict_path` extra param adds entries on top of whichever dictionary
+was selected, including the embedded default.
 
 ### Schema Definition
 

README_CN.md

@@ -16,7 +16,7 @@
 - **完善的错误处理** — 所有 FFI 调用返回 `Result<T>`，包含详细错误码
 - **尽可能零拷贝** — 最小化 FFI 边界的数据拷贝
 - **预编译库支持** — 自动从 GitHub Releases 下载预编译的 `libzvec_c_api`；高阶用户可通过 `ZVEC_LIB_DIR` 覆盖
-- **FTS jieba 开箱即用** — 预编译包内置 cppjieba 词表（`data/jieba_dict`）；`initialize` 自动发现词表并注册默认 jieba 词表目录，中文全文检索零配置
+- **FTS jieba 开箱即用** — `libzvec_c_api` 内嵌 cppjieba 默认词表与 HMM 模型（LZ4 压缩），`jieba` 分词器不再依赖磁盘上的词表文件，也无需注册步骤
 
 ## 支持的平台
 
@@ -242,8 +242,6 @@ cargo run --example vector_search
 | `shutdown()` | 释放所有资源 |
 | `version()` | 获取版本字符串 |
 | `is_initialized()` | 检查初始化状态 |
-| `set_default_jieba_dict_dir(dir)` | 设置进程级默认 jieba 词表目录（供 FTS `jieba` 分词器使用） |
-| `get_default_jieba_dict_dir()` | 获取当前默认 jieba 词表目录（未设置时为 `""`） |
 | `io_backend_type()` / `io_backend_type_name(code)` | 当前 DiskAnn I/O 后端类型码及名称（`io_uring`、`libaio`、`pread`、`windows_overlapped`） |
 | `io_backend_description()` | 当前 I/O 后端的可读描述 |
 
@@ -260,13 +258,12 @@ initialize(Some(&config))?;
 
 #### jieba 分词器词表
 
-FTS `jieba` 分词器需要 cppjieba 的词表文件（`jieba.dict.utf8` 与
-`hmm_model.utf8`）。预编译库已将词表放在 `libzvec_c_api` 同级的
-`data/jieba_dict` 目录下，`initialize` 会自动发现该目录并通过
-`zvec_set_default_jieba_dict_dir` 注册 —— 中文全文检索无需任何额外配置：
+`libzvec_c_api` 已内嵌 cppjieba 的默认词表与 HMM 模型（LZ4 压缩），
+FTS `jieba` 分词器直接从库中读取 —— 中文全文检索既不需要磁盘上的
+词表文件，也不需要任何额外配置：
 
 ```rust
-initialize(None)?; // 自动发现并注册 jieba 词表
+initialize(None)?;
 
 let schema = CollectionSchema::builder("articles")
     .add_field(FieldSchema::new("id", DataType::String, false, 0)?)
@@ -275,12 +272,14 @@ let schema = CollectionSchema::builder("articles")
     .build()?;
 ```
 
-如需自定义词表位置，可通过以下任一方式覆盖：
+如需改用自定义词表，可设置以下任一项（按优先级从高到低），
+目录中需包含 `jieba.dict.utf8` 与 `hmm_model.utf8`：
 
-- `ConfigBuilder::new().jieba_dict_dir("/path/to/jieba_dict")` —— 随 `initialize` 生效（优先级最高）
-- `set_default_jieba_dict_dir("/path/to/jieba_dict")` —— 进程级默认值
-- `ZVEC_JIEBA_DICT_DIR` 环境变量 —— 由 zvec 库在查询时读取
 - 字段级 `extra_params` JSON 中的 `"jieba_dict_dir"` —— 按索引生效
+- `ZVEC_JIEBA_DICT_DIR` 环境变量 —— 进程级，分词器初始化时读取
+
+`user_dict_path` 这个 extra param 会在已选定的词表（含内嵌默认词表）
+之上追加词条。
 
 ### Schema 定义
 

zvec-build/src/lib.rs

@@ -24,10 +24,9 @@
 //!    `cargo:rustc-link-arg`, which applies to every linked target kind of the
 //!    calling package — binaries, integration tests, examples, and benches — so
 //!    `cargo test` also runs without any dylib search-path environment vars.
-//! 2. Stages the shared library (and, when present, the `data/jieba_dict`
-//!    directory the FTS `jieba` tokenizer discovers relative to the loaded
-//!    library) next to the built artifacts in `target/<profile>/` so
-//!    `./target/<profile>/<exe>` runs with no environment variables set.
+//! 2. Stages the shared library next to the built artifacts in
+//!    `target/<profile>/` so `./target/<profile>/<exe>` runs with no
+//!    environment variables set.
 //!
 //! On Windows there is no rpath; the loader searches the executable's own
 //! directory, so step 2 (staging the DLL beside the executable) is what makes
@@ -86,22 +85,12 @@ pub fn runtime_lib_file_names(os: &str) -> &'static [&'static str] {
     }
 }
 
-/// Relative path (from the shared library's directory) of the cppjieba dict
-/// directory that the FTS `jieba` tokenizer discovers at runtime, and the two
-/// files it must contain.
-const JIEBA_DICT_SUBDIR: [&str; 2] = ["data", "jieba_dict"];
-const JIEBA_DICT_FILES: [&str; 2] = ["jieba.dict.utf8", "hmm_model.utf8"];
-
 /// Copies the runtime assets from the resolved [`lib_dir`] into `dst_dir`,
 /// creating directories as needed. Returns the destination paths that were
 /// written. A no-op returning an empty vec when the library cannot be located.
 ///
 /// Staged assets:
 /// - the runtime shared library ([`runtime_lib_file_names`]);
-/// - the `data/jieba_dict` directory, when present next to the library, so the
-///   FTS `jieba` tokenizer's runtime discovery (`<lib_dir>/data/jieba_dict`,
-///   located via `dladdr` / `GetModuleHandleEx`) also succeeds in a
-///   self-contained deployment.
 ///
 /// # Errors
 /// Returns any I/O error from creating a directory or copying a file.
@@ -119,7 +108,6 @@ pub fn copy_runtime_libs_to(dst_dir: &Path) -> io::Result<Vec<PathBuf>> {
             copied.push(destination);
         }
     }
-    copied.extend(copy_jieba_dict(&source_dir, dst_dir)?);
     Ok(copied)
 }
 
@@ -147,26 +135,6 @@ fn is_same_file(a: &Path, b: &Path) -> bool {
     }
 }
 
-/// Stages the `data/jieba_dict` directory from `source_dir` into `dst_dir`,
-/// preserving the relative layout the runtime discovery expects. No-op (empty
-/// vec) when the dict is not shipped alongside the library.
-fn copy_jieba_dict(source_dir: &Path, dst_dir: &Path) -> io::Result<Vec<PathBuf>> {
-    let subdir: PathBuf = JIEBA_DICT_SUBDIR.iter().collect();
-    let dict_src = source_dir.join(&subdir);
-    if !JIEBA_DICT_FILES.iter().all(|f| dict_src.join(f).is_file()) {
-        return Ok(Vec::new());
-    }
-    let dict_dst = dst_dir.join(&subdir);
-    fs::create_dir_all(&dict_dst)?;
-    let mut copied = Vec::new();
-    for name in JIEBA_DICT_FILES {
-        let destination = dict_dst.join(name);
-        stage_file(&dict_src.join(name), &destination)?;
-        copied.push(destination);
-    }
-    Ok(copied)
-}
-
 /// Directory that holds the crate's built binaries (`target/<profile>/`, or
 /// `target/<triple>/<profile>/` when cross-compiling), derived from `OUT_DIR`.
 ///

zvec-sys/build.rs

@@ -96,21 +96,6 @@ fn main() {
         }
     }
 
-    // Discover the cppjieba dict directory (jieba.dict.utf8 + hmm_model.utf8)
-    // near the resolved library so the high-level crate can auto-register the
-    // default jieba dict dir for the `jieba` FTS tokenizer.
-    if let Some(ref dir) = lib_dir {
-        if let Some(jieba_dir) =
-            resolve_jieba_dict_dir(&sibling_zvec, &submodule_zvec, &auto_build_dir, dir)
-        {
-            println!(
-                "cargo:rustc-env=ZVEC_RUST_JIEBA_DICT_DIR={}",
-                jieba_dir.display()
-            );
-            println!("cargo:rerun-if-changed={}", jieba_dir.display());
-        }
-    }
-
     println!("cargo:rustc-link-lib=dylib=zvec_c_api");
 }
 
@@ -646,39 +631,3 @@ fn num_cpus() -> String {
         .map(|s| s.trim().to_string())
         .unwrap_or_else(|| "2".to_string())
 }
-
-/// Locates the cppjieba dictionary directory for the resolved zvec library.
-///
-/// Candidates, in priority order:
-///   1. `lib_dir/data/jieba_dict` — prebuilt package layout (the release
-///      tarball ships the dict next to the shared library).
-///   2. `lib_dir/../data/jieba_dict` — native SDK install prefix layout
-///      (`<prefix>/lib` + `<prefix>/data`, as produced by zvec's CMake install).
-///   3. The `thirdparty/cppjieba` dict of a sibling / submodule / auto-built
-///      zvec source checkout.
-fn resolve_jieba_dict_dir(
-    sibling_zvec: &Option<PathBuf>,
-    submodule_zvec: &Path,
-    auto_build_dir: &Path,
-    lib_dir: &Path,
-) -> Option<PathBuf> {
-    let mut candidates: Vec<PathBuf> = vec![
-        lib_dir.join("data").join("jieba_dict"),
-        lib_dir.join("..").join("data").join("jieba_dict"),
-    ];
-    let dict_suffix = Path::new("thirdparty")
-        .join("cppjieba")
-        .join("cppjieba-5.6.7")
-        .join("dict");
-    if let Some(ref sibling) = sibling_zvec {
-        candidates.push(sibling.join(&dict_suffix));
-    }
-    candidates.push(submodule_zvec.join(&dict_suffix));
-    candidates.push(auto_build_dir.join("zvec").join(&dict_suffix));
-
-    candidates.into_iter().find(|c| has_jieba_dict(c))
-}
-
-fn has_jieba_dict(dir: &Path) -> bool {
-    dir.join("jieba.dict.utf8").is_file() && dir.join("hmm_model.utf8").is_file()
-}

zvec-sys/src/lib.rs

@@ -619,11 +619,6 @@ extern "C" {
         thread_count: u32,
     ) -> zvec_error_code_t;
     pub fn zvec_config_data_get_optimize_thread_count(config: *const zvec_config_data_t) -> u32;
-    pub fn zvec_config_data_set_jieba_dict_dir(
-        config: *mut zvec_config_data_t,
-        dir: *const c_char,
-    ) -> zvec_error_code_t;
-    pub fn zvec_config_data_get_jieba_dict_dir(config: *const zvec_config_data_t) -> *const c_char;
 
     // -------------------------------------------------------------------------
     // Initialization
@@ -632,8 +627,6 @@ extern "C" {
     pub fn zvec_initialize(config: *const zvec_config_data_t) -> zvec_error_code_t;
     pub fn zvec_shutdown() -> zvec_error_code_t;
     pub fn zvec_is_initialized() -> bool;
-    pub fn zvec_set_default_jieba_dict_dir(dir: *const c_char);
-    pub fn zvec_get_default_jieba_dict_dir() -> *const c_char;
 
     // -------------------------------------------------------------------------
     // I/O Backend Introspection
@@ -1770,13 +1763,3 @@ extern "C" {
     pub fn zvec_index_type_to_string(index_type: zvec_index_type_t) -> *const c_char;
     pub fn zvec_metric_type_to_string(metric_type: zvec_metric_type_t) -> *const c_char;
 }
-
-/// Returns the jieba dict directory discovered at build time, if any.
-///
-/// The build script looks for the cppjieba dictionary files next to the
-/// resolved zvec library (prebuilt `data/jieba_dict/`, an SDK install prefix,
-/// or a zvec source checkout) and records the path here. The high-level
-/// `zvec` crate uses this to auto-register the default jieba dict dir.
-pub fn build_time_jieba_dict_dir() -> Option<&'static str> {
-    option_env!("ZVEC_RUST_JIEBA_DICT_DIR")
-}

zvec/build.rs

@@ -1,13 +1,6 @@
 use std::env;
 
 fn main() {
-    // The jieba dict auto-discovery uses dladdr() to locate the loaded
-    // libzvec_c_api at runtime. On Linux dladdr lives in libdl (merged into
-    // libc on glibc >= 2.34, but -ldl remains valid there and on musl).
-    if env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("linux") {
-        println!("cargo:rustc-link-lib=dylib=dl");
-    }
-
     // Forward the native library directory resolved by `zvec-rust-sys` one hop
     // further down the graph. `zvec-rust-sys` (links = "zvec_c_api") exposes it
     // to this build script as `DEP_ZVEC_C_API_LIB_DIR`; by re-publishing it as