diff --git a/.github/workflows/build-prebuilt.yml b/.github/workflows/build-prebuilt.yml index ecc8975..384d316 100644 --- a/.github/workflows/build-prebuilt.yml +++ b/.github/workflows/build-prebuilt.yml @@ -250,17 +250,9 @@ jobs: cp "$ZVEC_LIB_DIR/${{ matrix.lib_name }}" dist/ # Also copy any symlinks or versioned libraries find "$ZVEC_LIB_DIR" -name 'libzvec_c_api*' -exec cp -P {} dist/ \; 2>/dev/null || true - # Bundle cppjieba's dictionary files so the `jieba` FTS tokenizer - # works out of the box (same layout as the upstream zvec SDK: - # data/jieba_dict next to the library). - mkdir -p dist/data/jieba_dict - cp vendor/zvec/thirdparty/cppjieba/cppjieba-5.6.7/dict/jieba.dict.utf8 \ - vendor/zvec/thirdparty/cppjieba/cppjieba-5.6.7/dict/hmm_model.utf8 \ - dist/data/jieba_dict/ echo "${{ matrix.target }}" > dist/TARGET tar czf zvec-prebuilt-${{ matrix.target }}.tar.gz -C dist . ls -lh zvec-prebuilt-${{ matrix.target }}.tar.gz - ls -l dist/data/jieba_dict shell: bash - name: Package library (Windows) @@ -272,17 +264,9 @@ jobs: find vendor/zvec/build -name 'zvec_c_api.lib' -exec cp {} dist/ \; 2>/dev/null || true find vendor/zvec/build -name 'zvec_c_api.dll' -path '*/Release/*' -exec cp {} dist/ \; 2>/dev/null || true find vendor/zvec/build -name 'zvec_c_api.lib' -path '*/Release/*' -exec cp {} dist/ \; 2>/dev/null || true - # Bundle cppjieba's dictionary files so the `jieba` FTS tokenizer - # works out of the box (same layout as the upstream zvec SDK: - # data/jieba_dict next to the library). - mkdir -p dist/data/jieba_dict - cp vendor/zvec/thirdparty/cppjieba/cppjieba-5.6.7/dict/jieba.dict.utf8 \ - vendor/zvec/thirdparty/cppjieba/cppjieba-5.6.7/dict/hmm_model.utf8 \ - dist/data/jieba_dict/ echo "${{ matrix.target }}" > dist/TARGET tar czf zvec-prebuilt-${{ matrix.target }}.tar.gz -C dist . ls -lh zvec-prebuilt-${{ matrix.target }}.tar.gz - ls -l dist/data/jieba_dict shell: bash - name: Upload artifact diff --git a/README.md b/README.md index 7c042f7..f8b9b03 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Safe, idiomatic Rust bindings for the [zvec](https://github.com/alibaba/zvec) ve - **Comprehensive Error Handling** — All FFI calls return `Result` with detailed error codes - **Zero-Copy Where Possible** — Minimizes data copying across the FFI boundary - **Prebuilt Libraries** — Automatically downloads prebuilt `libzvec_c_api` from GitHub Releases; advanced users can override with `ZVEC_LIB_DIR` -- **FTS with jieba Out of the Box** — Prebuilt packages ship the cppjieba dictionary (`data/jieba_dict`); `initialize` auto-discovers it and registers the default jieba dict dir for the `jieba` FTS tokenizer +- **FTS with jieba Out of the Box** — `libzvec_c_api` embeds cppjieba's default dictionary and HMM model (LZ4-compressed), so the `jieba` FTS tokenizer needs no dictionary files on disk and no registration step ## Supported Platforms @@ -250,8 +250,6 @@ cargo run --example vector_search | `shutdown()` | Release all resources | | `version()` | Get version string | | `is_initialized()` | Check initialization status | -| `set_default_jieba_dict_dir(dir)` | Set the process-wide default jieba dict dir for the FTS `jieba` tokenizer | -| `get_default_jieba_dict_dir()` | Get the current default jieba dict dir (`""` when unset) | | `io_backend_type()` / `io_backend_type_name(code)` | Active DiskAnn I/O backend code and name (`io_uring`, `libaio`, `pread`, `windows_overlapped`) | | `io_backend_description()` | Human-readable description of the active I/O backend | @@ -268,14 +266,13 @@ initialize(Some(&config))?; #### jieba FTS tokenizer dictionary -The `jieba` FTS tokenizer needs cppjieba's dictionary files (`jieba.dict.utf8` -and `hmm_model.utf8`). The prebuilt libraries ship them under `data/jieba_dict` -next to `libzvec_c_api`, and `initialize` automatically discovers that -directory and registers it via `zvec_set_default_jieba_dict_dir` — Chinese -full-text search works with no extra configuration: +`libzvec_c_api` embeds cppjieba's default dictionary and HMM model +(LZ4-compressed), so the `jieba` FTS tokenizer reads them from the library +itself — Chinese full-text search works with no dictionary files on disk and +no extra configuration: ```rust -initialize(None)?; // jieba dict auto-discovered and registered +initialize(None)?; let schema = CollectionSchema::builder("articles") .add_field(FieldSchema::new("id", DataType::String, false, 0)?) @@ -284,12 +281,15 @@ let schema = CollectionSchema::builder("articles") .build()?; ``` -To override the location (e.g. a custom dictionary), use one of: +To use a custom dictionary instead of the embedded default, set one of the +following (highest priority first); the directory must contain +`jieba.dict.utf8` and `hmm_model.utf8`: -- `ConfigBuilder::new().jieba_dict_dir("/path/to/jieba_dict")` — per `initialize` call (highest priority) -- `set_default_jieba_dict_dir("/path/to/jieba_dict")` — process-wide default -- `ZVEC_JIEBA_DICT_DIR` environment variable — read by the zvec library at query time - per-field `extra_params` JSON with `"jieba_dict_dir"` — per index +- `ZVEC_JIEBA_DICT_DIR` environment variable — process-wide, read when the tokenizer initializes + +The `user_dict_path` extra param adds entries on top of whichever dictionary +was selected, including the embedded default. ### Schema Definition diff --git a/README_CN.md b/README_CN.md index 9bf91bc..6c07c6d 100644 --- a/README_CN.md +++ b/README_CN.md @@ -16,7 +16,7 @@ - **完善的错误处理** — 所有 FFI 调用返回 `Result`，包含详细错误码 - **尽可能零拷贝** — 最小化 FFI 边界的数据拷贝 - **预编译库支持** — 自动从 GitHub Releases 下载预编译的 `libzvec_c_api`；高阶用户可通过 `ZVEC_LIB_DIR` 覆盖 -- **FTS jieba 开箱即用** — 预编译包内置 cppjieba 词表（`data/jieba_dict`）；`initialize` 自动发现词表并注册默认 jieba 词表目录，中文全文检索零配置 +- **FTS jieba 开箱即用** — `libzvec_c_api` 内嵌 cppjieba 默认词表与 HMM 模型（LZ4 压缩），`jieba` 分词器不再依赖磁盘上的词表文件，也无需注册步骤 ## 支持的平台 @@ -242,8 +242,6 @@ cargo run --example vector_search | `shutdown()` | 释放所有资源 | | `version()` | 获取版本字符串 | | `is_initialized()` | 检查初始化状态 | -| `set_default_jieba_dict_dir(dir)` | 设置进程级默认 jieba 词表目录（供 FTS `jieba` 分词器使用） | -| `get_default_jieba_dict_dir()` | 获取当前默认 jieba 词表目录（未设置时为 `""`） | | `io_backend_type()` / `io_backend_type_name(code)` | 当前 DiskAnn I/O 后端类型码及名称（`io_uring`、`libaio`、`pread`、`windows_overlapped`） | | `io_backend_description()` | 当前 I/O 后端的可读描述 | @@ -260,13 +258,12 @@ initialize(Some(&config))?; #### jieba 分词器词表 -FTS `jieba` 分词器需要 cppjieba 的词表文件（`jieba.dict.utf8` 与 -`hmm_model.utf8`）。预编译库已将词表放在 `libzvec_c_api` 同级的 -`data/jieba_dict` 目录下，`initialize` 会自动发现该目录并通过 -`zvec_set_default_jieba_dict_dir` 注册 —— 中文全文检索无需任何额外配置： +`libzvec_c_api` 已内嵌 cppjieba 的默认词表与 HMM 模型（LZ4 压缩）， +FTS `jieba` 分词器直接从库中读取 —— 中文全文检索既不需要磁盘上的 +词表文件，也不需要任何额外配置： ```rust -initialize(None)?; // 自动发现并注册 jieba 词表 +initialize(None)?; let schema = CollectionSchema::builder("articles") .add_field(FieldSchema::new("id", DataType::String, false, 0)?) @@ -275,12 +272,14 @@ let schema = CollectionSchema::builder("articles") .build()?; ``` -如需自定义词表位置，可通过以下任一方式覆盖： +如需改用自定义词表，可设置以下任一项（按优先级从高到低）， +目录中需包含 `jieba.dict.utf8` 与 `hmm_model.utf8`： -- `ConfigBuilder::new().jieba_dict_dir("/path/to/jieba_dict")` —— 随 `initialize` 生效（优先级最高） -- `set_default_jieba_dict_dir("/path/to/jieba_dict")` —— 进程级默认值 -- `ZVEC_JIEBA_DICT_DIR` 环境变量 —— 由 zvec 库在查询时读取 - 字段级 `extra_params` JSON 中的 `"jieba_dict_dir"` —— 按索引生效 +- `ZVEC_JIEBA_DICT_DIR` 环境变量 —— 进程级，分词器初始化时读取 + +`user_dict_path` 这个 extra param 会在已选定的词表（含内嵌默认词表） +之上追加词条。 ### Schema 定义 diff --git a/vendor/zvec b/vendor/zvec index 60956b1..c133f20 160000 --- a/vendor/zvec +++ b/vendor/zvec @@ -1 +1 @@ -Subproject commit 60956b127e1e9cf7494741004da619acc528901c +Subproject commit c133f2054c5dd30dd5a8bd4b50f53b6e6da863d2 diff --git a/zvec-build/src/lib.rs b/zvec-build/src/lib.rs index cde857a..60e80f7 100644 --- a/zvec-build/src/lib.rs +++ b/zvec-build/src/lib.rs @@ -24,10 +24,9 @@ //! `cargo:rustc-link-arg`, which applies to every linked target kind of the //! calling package — binaries, integration tests, examples, and benches — so //! `cargo test` also runs without any dylib search-path environment vars. -//! 2. Stages the shared library (and, when present, the `data/jieba_dict` -//! directory the FTS `jieba` tokenizer discovers relative to the loaded -//! library) next to the built artifacts in `target//` so -//! `./target//` runs with no environment variables set. +//! 2. Stages the shared library next to the built artifacts in +//! `target//` so `./target//` runs with no +//! environment variables set. //! //! On Windows there is no rpath; the loader searches the executable's own //! directory, so step 2 (staging the DLL beside the executable) is what makes @@ -86,22 +85,12 @@ pub fn runtime_lib_file_names(os: &str) -> &'static [&'static str] { } } -/// Relative path (from the shared library's directory) of the cppjieba dict -/// directory that the FTS `jieba` tokenizer discovers at runtime, and the two -/// files it must contain. -const JIEBA_DICT_SUBDIR: [&str; 2] = ["data", "jieba_dict"]; -const JIEBA_DICT_FILES: [&str; 2] = ["jieba.dict.utf8", "hmm_model.utf8"]; - /// Copies the runtime assets from the resolved [`lib_dir`] into `dst_dir`, /// creating directories as needed. Returns the destination paths that were /// written. A no-op returning an empty vec when the library cannot be located. /// /// Staged assets: /// - the runtime shared library ([`runtime_lib_file_names`]); -/// - the `data/jieba_dict` directory, when present next to the library, so the -/// FTS `jieba` tokenizer's runtime discovery (`/data/jieba_dict`, -/// located via `dladdr` / `GetModuleHandleEx`) also succeeds in a -/// self-contained deployment. /// /// # Errors /// Returns any I/O error from creating a directory or copying a file. @@ -119,7 +108,6 @@ pub fn copy_runtime_libs_to(dst_dir: &Path) -> io::Result> { copied.push(destination); } } - copied.extend(copy_jieba_dict(&source_dir, dst_dir)?); Ok(copied) } @@ -147,26 +135,6 @@ fn is_same_file(a: &Path, b: &Path) -> bool { } } -/// Stages the `data/jieba_dict` directory from `source_dir` into `dst_dir`, -/// preserving the relative layout the runtime discovery expects. No-op (empty -/// vec) when the dict is not shipped alongside the library. -fn copy_jieba_dict(source_dir: &Path, dst_dir: &Path) -> io::Result> { - let subdir: PathBuf = JIEBA_DICT_SUBDIR.iter().collect(); - let dict_src = source_dir.join(&subdir); - if !JIEBA_DICT_FILES.iter().all(|f| dict_src.join(f).is_file()) { - return Ok(Vec::new()); - } - let dict_dst = dst_dir.join(&subdir); - fs::create_dir_all(&dict_dst)?; - let mut copied = Vec::new(); - for name in JIEBA_DICT_FILES { - let destination = dict_dst.join(name); - stage_file(&dict_src.join(name), &destination)?; - copied.push(destination); - } - Ok(copied) -} - /// Directory that holds the crate's built binaries (`target//`, or /// `target///` when cross-compiling), derived from `OUT_DIR`. /// diff --git a/zvec-sys/build.rs b/zvec-sys/build.rs index 0cd7d0b..bc372c3 100644 --- a/zvec-sys/build.rs +++ b/zvec-sys/build.rs @@ -96,21 +96,6 @@ fn main() { } } - // Discover the cppjieba dict directory (jieba.dict.utf8 + hmm_model.utf8) - // near the resolved library so the high-level crate can auto-register the - // default jieba dict dir for the `jieba` FTS tokenizer. - if let Some(ref dir) = lib_dir { - if let Some(jieba_dir) = - resolve_jieba_dict_dir(&sibling_zvec, &submodule_zvec, &auto_build_dir, dir) - { - println!( - "cargo:rustc-env=ZVEC_RUST_JIEBA_DICT_DIR={}", - jieba_dir.display() - ); - println!("cargo:rerun-if-changed={}", jieba_dir.display()); - } - } - println!("cargo:rustc-link-lib=dylib=zvec_c_api"); } @@ -646,39 +631,3 @@ fn num_cpus() -> String { .map(|s| s.trim().to_string()) .unwrap_or_else(|| "2".to_string()) } - -/// Locates the cppjieba dictionary directory for the resolved zvec library. -/// -/// Candidates, in priority order: -/// 1. `lib_dir/data/jieba_dict` — prebuilt package layout (the release -/// tarball ships the dict next to the shared library). -/// 2. `lib_dir/../data/jieba_dict` — native SDK install prefix layout -/// (`/lib` + `/data`, as produced by zvec's CMake install). -/// 3. The `thirdparty/cppjieba` dict of a sibling / submodule / auto-built -/// zvec source checkout. -fn resolve_jieba_dict_dir( - sibling_zvec: &Option, - submodule_zvec: &Path, - auto_build_dir: &Path, - lib_dir: &Path, -) -> Option { - let mut candidates: Vec = vec![ - lib_dir.join("data").join("jieba_dict"), - lib_dir.join("..").join("data").join("jieba_dict"), - ]; - let dict_suffix = Path::new("thirdparty") - .join("cppjieba") - .join("cppjieba-5.6.7") - .join("dict"); - if let Some(ref sibling) = sibling_zvec { - candidates.push(sibling.join(&dict_suffix)); - } - candidates.push(submodule_zvec.join(&dict_suffix)); - candidates.push(auto_build_dir.join("zvec").join(&dict_suffix)); - - candidates.into_iter().find(|c| has_jieba_dict(c)) -} - -fn has_jieba_dict(dir: &Path) -> bool { - dir.join("jieba.dict.utf8").is_file() && dir.join("hmm_model.utf8").is_file() -} diff --git a/zvec-sys/src/lib.rs b/zvec-sys/src/lib.rs index b7d6fb7..32eaae3 100644 --- a/zvec-sys/src/lib.rs +++ b/zvec-sys/src/lib.rs @@ -619,11 +619,6 @@ extern "C" { thread_count: u32, ) -> zvec_error_code_t; pub fn zvec_config_data_get_optimize_thread_count(config: *const zvec_config_data_t) -> u32; - pub fn zvec_config_data_set_jieba_dict_dir( - config: *mut zvec_config_data_t, - dir: *const c_char, - ) -> zvec_error_code_t; - pub fn zvec_config_data_get_jieba_dict_dir(config: *const zvec_config_data_t) -> *const c_char; // ------------------------------------------------------------------------- // Initialization @@ -632,8 +627,6 @@ extern "C" { pub fn zvec_initialize(config: *const zvec_config_data_t) -> zvec_error_code_t; pub fn zvec_shutdown() -> zvec_error_code_t; pub fn zvec_is_initialized() -> bool; - pub fn zvec_set_default_jieba_dict_dir(dir: *const c_char); - pub fn zvec_get_default_jieba_dict_dir() -> *const c_char; // ------------------------------------------------------------------------- // I/O Backend Introspection @@ -1770,13 +1763,3 @@ extern "C" { pub fn zvec_index_type_to_string(index_type: zvec_index_type_t) -> *const c_char; pub fn zvec_metric_type_to_string(metric_type: zvec_metric_type_t) -> *const c_char; } - -/// Returns the jieba dict directory discovered at build time, if any. -/// -/// The build script looks for the cppjieba dictionary files next to the -/// resolved zvec library (prebuilt `data/jieba_dict/`, an SDK install prefix, -/// or a zvec source checkout) and records the path here. The high-level -/// `zvec` crate uses this to auto-register the default jieba dict dir. -pub fn build_time_jieba_dict_dir() -> Option<&'static str> { - option_env!("ZVEC_RUST_JIEBA_DICT_DIR") -} diff --git a/zvec/build.rs b/zvec/build.rs index 24420bb..1c6f6e0 100644 --- a/zvec/build.rs +++ b/zvec/build.rs @@ -1,13 +1,6 @@ use std::env; fn main() { - // The jieba dict auto-discovery uses dladdr() to locate the loaded - // libzvec_c_api at runtime. On Linux dladdr lives in libdl (merged into - // libc on glibc >= 2.34, but -ldl remains valid there and on musl). - if env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("linux") { - println!("cargo:rustc-link-lib=dylib=dl"); - } - // Forward the native library directory resolved by `zvec-rust-sys` one hop // further down the graph. `zvec-rust-sys` (links = "zvec_c_api") exposes it // to this build script as `DEP_ZVEC_C_API_LIB_DIR`; by re-publishing it as diff --git a/zvec/src/config.rs b/zvec/src/config.rs index 53df28e..3a98596 100644 --- a/zvec/src/config.rs +++ b/zvec/src/config.rs @@ -1,7 +1,6 @@ use std::ffi::CStr; -use std::path::{Path, PathBuf}; -use crate::error::{check_error, to_cstring, Error, ErrorCode, Result}; +use crate::error::{check_error, Error, ErrorCode, Result}; use crate::types::LogLevel; /// Configuration builder for initializing the zvec library. @@ -28,9 +27,6 @@ pub struct ConfigBuilder { pub enable_console_log: bool, /// FTS brute-force-by-keys ratio (None = use library default). pub fts_brute_force_by_keys_ratio: Option, - /// Jieba dict directory for the FTS `jieba` tokenizer - /// (None = auto-discover and register the process-wide default). - pub jieba_dict_dir: Option, } impl ConfigBuilder { @@ -41,7 +37,6 @@ impl ConfigBuilder { num_threads: 0, enable_console_log: false, fts_brute_force_by_keys_ratio: None, - jieba_dict_dir: None, } } @@ -69,18 +64,6 @@ impl ConfigBuilder { self } - /// Sets the jieba dict directory for the FTS `jieba` tokenizer. - /// - /// The directory must contain `jieba.dict.utf8` and `hmm_model.utf8` - /// (shipped with the zvec prebuilt packages under `data/jieba_dict`). - /// When unset, [`initialize`] auto-discovers the dict directory that - /// ships with the zvec library and registers it as the process-wide - /// default via [`set_default_jieba_dict_dir`]. - pub fn jieba_dict_dir(mut self, dir: impl Into) -> Self { - self.jieba_dict_dir = Some(dir.into()); - self - } - /// Finalizes the builder configuration. /// /// This is a no-op that returns `self` for API consistency. The builder @@ -136,13 +119,6 @@ impl ConfigData { }) } - pub(crate) fn set_jieba_dict_dir(&mut self, dir: &str) -> Result<()> { - let c_dir = to_cstring(dir)?; - check_error(unsafe { - zvec_rust_sys::zvec_config_data_set_jieba_dict_dir(self.handle, c_dir.as_ptr()) - }) - } - pub(crate) fn set_console_log(&mut self, level: LogLevel) -> Result<()> { let log_config = unsafe { zvec_rust_sys::zvec_config_log_create_console(level as u32) }; if log_config.is_null() { @@ -195,10 +171,7 @@ impl Drop for ConfigData { /// ``` pub fn initialize(config: Option<&ConfigBuilder>) -> Result<()> { match config { - None => { - auto_register_default_jieba_dict_dir(); - check_error(unsafe { zvec_rust_sys::zvec_initialize(std::ptr::null()) }) - } + None => check_error(unsafe { zvec_rust_sys::zvec_initialize(std::ptr::null()) }), Some(builder) => { let mut cfg = ConfigData::new()?; if builder.memory_limit > 0 { @@ -214,172 +187,11 @@ pub fn initialize(config: Option<&ConfigBuilder>) -> Result<()> { if let Some(ratio) = builder.fts_brute_force_by_keys_ratio { cfg.set_fts_brute_force_by_keys_ratio(ratio)?; } - if let Some(ref dir) = builder.jieba_dict_dir { - cfg.set_jieba_dict_dir(dir)?; - } else { - auto_register_default_jieba_dict_dir(); - } check_error(unsafe { zvec_rust_sys::zvec_initialize(cfg.handle as *const _) }) } } } -/// Sets the process-wide default jieba dict directory. -/// -/// The directory must contain `jieba.dict.utf8` and `hmm_model.utf8` (as -/// shipped with the zvec prebuilt packages under `data/jieba_dict`). The FTS -/// `jieba` tokenizer uses this default when neither the per-field -/// `jieba_dict_dir` extra param nor the `ZVEC_JIEBA_DICT_DIR` environment -/// variable is set. Passing an empty string clears the default. -/// -/// Thread-safe; last writer wins. A subsequent [`initialize`] with a -/// non-empty [`ConfigBuilder::jieba_dict_dir`] overrides this default. -pub fn set_default_jieba_dict_dir(dir: &str) -> Result<()> { - let c_dir = to_cstring(dir)?; - unsafe { zvec_rust_sys::zvec_set_default_jieba_dict_dir(c_dir.as_ptr()) }; - Ok(()) -} - -/// Returns the process-wide default jieba dict directory, or `""` if unset. -pub fn get_default_jieba_dict_dir() -> String { - unsafe { - let ptr = zvec_rust_sys::zvec_get_default_jieba_dict_dir(); - if ptr.is_null() { - return String::new(); - } - CStr::from_ptr(ptr).to_string_lossy().into_owned() - } -} - -/// Registers an auto-discovered jieba dict directory as the process-wide -/// default, unless one is already registered. -/// -/// Called automatically by [`initialize`] so the `jieba` FTS tokenizer works -/// out of the box with the bundled prebuilt libraries. -fn auto_register_default_jieba_dict_dir() { - if !get_default_jieba_dict_dir().is_empty() { - return; - } - if let Some(dir) = discover_jieba_dict_dir() { - let dir = dir.to_string_lossy().into_owned(); - let _ = set_default_jieba_dict_dir(&dir); - } -} - -/// Searches known locations for a directory containing `jieba.dict.utf8` -/// and `hmm_model.utf8`, returning the first hit. -fn discover_jieba_dict_dir() -> Option { - let mut candidates: Vec = Vec::new(); - - // 1. Path recorded at build time by zvec-rust-sys (prebuilt download - // cache, sibling/submodule checkout, or ZVEC_LIB_DIR). - if let Some(build_time_dir) = zvec_rust_sys::build_time_jieba_dict_dir() { - candidates.push(PathBuf::from(build_time_dir)); - } - - // 2. Relative to the loaded libzvec_c_api: prebuilt packages ship the - // dict at `/data/jieba_dict`, native SDK install prefixes at - // `/data/jieba_dict` next to `/lib`. - if let Some(lib_dir) = loaded_library_dir() { - candidates.push(lib_dir.join("data").join("jieba_dict")); - candidates.push(lib_dir.join("..").join("data").join("jieba_dict")); - } - - candidates.into_iter().find(|dir| is_jieba_dict_dir(dir)) -} - -fn is_jieba_dict_dir(dir: &Path) -> bool { - dir.join("jieba.dict.utf8").is_file() && dir.join("hmm_model.utf8").is_file() -} - -/// Directory of the loaded `libzvec_c_api` shared library, if determinable. -#[cfg(unix)] -fn loaded_library_dir() -> Option { - #[repr(C)] - struct DlInfo { - dli_fname: *const std::ffi::c_char, - dli_fbase: *mut std::ffi::c_void, - dli_sname: *const std::ffi::c_char, - dli_saddr: *mut std::ffi::c_void, - } - extern "C" { - fn dladdr(addr: *const std::ffi::c_void, info: *mut DlInfo) -> std::ffi::c_int; - } - - // Probe with a symbol that was definitely called already: with lazy - // binding the GOT slot only holds the resolved in-library address after - // the first call; an uncalled symbol could still point at the PLT stub - // inside the executable. - let symbol = - zvec_rust_sys::zvec_get_default_jieba_dict_dir as *const () as *const std::ffi::c_void; - - let mut info = DlInfo { - dli_fname: std::ptr::null(), - dli_fbase: std::ptr::null_mut(), - dli_sname: std::ptr::null(), - dli_saddr: std::ptr::null_mut(), - }; - let path = unsafe { - if dladdr(symbol, &mut info) == 0 || info.dli_fname.is_null() { - return None; - } - PathBuf::from(CStr::from_ptr(info.dli_fname).to_string_lossy().as_ref()) - }; - let resolved = std::fs::canonicalize(&path).unwrap_or(path); - resolved.parent().map(PathBuf::from) -} - -/// Directory of the loaded `zvec_c_api.dll`, if determinable. -#[cfg(windows)] -fn loaded_library_dir() -> Option { - use std::os::windows::ffi::OsStringExt; - - type HModule = *mut std::ffi::c_void; - const FROM_ADDRESS: u32 = 0x0000_0004; - const UNCHANGED_REFCOUNT: u32 = 0x0000_0002; - - #[link(name = "kernel32")] - extern "system" { - fn GetModuleHandleExW( - flags: u32, - module_name: *mut std::ffi::c_void, - module: *mut HModule, - ) -> i32; - fn GetModuleFileNameW(module: HModule, filename: *mut u16, size: u32) -> u32; - } - - let symbol = - zvec_rust_sys::zvec_get_default_jieba_dict_dir as *const () as *mut std::ffi::c_void; - unsafe { - let mut module: HModule = std::ptr::null_mut(); - if GetModuleHandleExW(FROM_ADDRESS | UNCHANGED_REFCOUNT, symbol, &mut module) == 0 - || module.is_null() - { - return None; - } - let mut size = 260u32; // MAX_PATH - loop { - let mut buf = vec![0u16; size as usize]; - let len = GetModuleFileNameW(module, buf.as_mut_ptr(), size); - if len == 0 { - return None; - } - if len < size { - buf.truncate(len as usize); - return PathBuf::from(std::ffi::OsString::from_wide(&buf)) - .parent() - .map(PathBuf::from); - } - size = size.checked_mul(2)?; // buffer too small; grow and retry - } - } -} - -#[cfg(not(any(unix, windows)))] -fn loaded_library_dir() -> Option { - None -} - /// Shuts down the zvec library and releases all resources. #[doc(hidden)] pub fn shutdown() -> Result<()> { @@ -471,7 +283,6 @@ mod tests { assert_eq!(builder.memory_limit, 0); assert_eq!(builder.num_threads, 0); assert!(!builder.enable_console_log); - assert!(builder.jieba_dict_dir.is_none()); } #[test] @@ -512,37 +323,6 @@ mod tests { assert!(builder.enable_console_log); } - #[test] - fn config_builder_jieba_dict_dir_setter() { - let builder = ConfigBuilder::new().jieba_dict_dir("/tmp/jieba_dict"); - assert_eq!(builder.jieba_dict_dir.as_deref(), Some("/tmp/jieba_dict")); - } - - #[test] - fn default_jieba_dict_dir_roundtrip() { - let previous = get_default_jieba_dict_dir(); - set_default_jieba_dict_dir("/tmp/zvec_rust_jieba_roundtrip").unwrap(); - assert_eq!( - get_default_jieba_dict_dir(), - "/tmp/zvec_rust_jieba_roundtrip" - ); - // Empty string clears; restore the previous value afterwards so the - // process-wide default is left as found. - set_default_jieba_dict_dir("").unwrap(); - assert_eq!(get_default_jieba_dict_dir(), ""); - set_default_jieba_dict_dir(&previous).unwrap(); - assert_eq!(get_default_jieba_dict_dir(), previous); - } - - #[test] - fn discovered_jieba_dict_dir_contains_dict_files() { - // Discovery is environment-dependent (build-time path or dylib - // location); when it finds something, it must be a valid dict dir. - if let Some(dir) = discover_jieba_dict_dir() { - assert!(is_jieba_dict_dir(&dir), "invalid dict dir: {:?}", dir); - } - } - #[test] fn config_builder_build_returns_self() { let builder = ConfigBuilder::new() @@ -591,19 +371,6 @@ mod tests { assert_eq!(builder.fts_brute_force_by_keys_ratio, Some(0.5)); } - #[test] - fn loaded_library_dir_resolves_zvec_library() { - // dladdr / GetModuleHandleEx based discovery of libzvec_c_api. - if let Some(dir) = loaded_library_dir() { - let dir_str = dir.to_string_lossy(); - assert!( - dir_str.contains("zvec"), - "unexpected dylib dir: {}", - dir_str - ); - } - } - #[test] fn io_backend_type_name_maps_every_code() { use zvec_rust_sys::{ diff --git a/zvec/src/lib.rs b/zvec/src/lib.rs index 734c6fe..9ee3f8f 100644 --- a/zvec/src/lib.rs +++ b/zvec/src/lib.rs @@ -50,9 +50,8 @@ pub use collection::{ WriteResult, }; pub use config::{ - get_default_jieba_dict_dir, initialize, io_backend_description, io_backend_type, - io_backend_type_name, is_initialized, set_default_jieba_dict_dir, shutdown, version, - ConfigBuilder, + initialize, io_backend_description, io_backend_type, io_backend_type_name, is_initialized, + shutdown, version, ConfigBuilder, }; pub use doc::Doc; pub use error::{Error, ErrorCode, Result}; diff --git a/zvec/tests/jieba_fts_test.rs b/zvec/tests/jieba_fts_test.rs index 1983a36..7407e79 100644 --- a/zvec/tests/jieba_fts_test.rs +++ b/zvec/tests/jieba_fts_test.rs @@ -1,10 +1,11 @@ -//! End-to-end test for the `jieba` FTS tokenizer and the auto-registration -//! of the default jieba dict directory. +//! End-to-end test for the `jieba` FTS tokenizer. //! -//! The zvec prebuilt packages ship cppjieba's dictionary under -//! `data/jieba_dict`; `initialize` must discover it and register it via -//! `zvec_set_default_jieba_dict_dir` so Chinese full-text search works -//! without any extra configuration. +//! zvec embeds cppjieba's default dictionary and HMM model (LZ4-compressed) +//! in `libzvec_c_api`, so Chinese full-text search needs no dict directory +//! to ship, discover, or register: `initialize(None)` plus +//! `IndexParams::fts(Some("jieba"), ..)` is enough. A custom dictionary can +//! still be selected per field through the `jieba_dict_dir` extra param, or +//! process-wide through the `ZVEC_JIEBA_DICT_DIR` environment variable. //! //! Requires the zvec C library (`libzvec_c_api`). Set `ZVEC_LIB_DIR` before //! running. Run with: `cargo test --test jieba_fts_test` @@ -20,40 +21,10 @@ fn ensure_initialized() { }); } -#[test] -fn initialize_auto_registers_default_jieba_dict_dir() { - ensure_initialized(); - - let dir = get_default_jieba_dict_dir(); - if dir.is_empty() { - // Soft check: discovery depends on the dict shipping with the - // resolved zvec library (e.g. prebuilt assets published before the - // jieba dict was packaged do not have it). - eprintln!("skipping: no jieba dict discovered next to libzvec_c_api"); - return; - } - let path = std::path::Path::new(&dir); - assert!( - path.join("jieba.dict.utf8").is_file(), - "missing jieba.dict.utf8 in {}", - dir - ); - assert!( - path.join("hmm_model.utf8").is_file(), - "missing hmm_model.utf8 in {}", - dir - ); -} - #[test] fn jieba_tokenizer_search_chinese_text() { ensure_initialized(); - if get_default_jieba_dict_dir().is_empty() { - eprintln!("skipping: no jieba dict discovered next to libzvec_c_api"); - return; - } - let tmp_dir = tempfile::tempdir().unwrap(); let dir = tmp_dir.path().join("zvec_jieba");