Materialize HNSW Layered Index to hnswlib Index

c/include/cuvs/neighbors/hnsw.h

@@ -594,6 +594,81 @@ CUVS_EXPORT cuvsError_t cuvsHnswDeserialize(cuvsResources_t res,
  * @}
  */
 
+/**
+ * @defgroup hnsw_c_index_materialize Materialize a layered HNSW artifact to an hnswlib index
+ * @{
+ */
+
+/**
+ * @brief Parameters for materializing a layered HNSW artifact into an hnswlib index on disk.
+ */
+struct cuvsHnswMaterializeParams {
+  /**
+   * Local dataset path holding the original-ID-ordered vectors used to build the artifact.
+   *
+   * Supported formats match layered deserialization: `.npy` and ANN benchmark `*.bin` files with a
+   * `[uint32 rows, uint32 cols]` header (`.fbin`, `.f16bin`, `.u8bin`, `.i8bin`).
+   */
+  const char* dataset_path;
+  /**
+   * Upper bound on host memory (in GiB) used for the base-topology reorder buffer.
+   *
+   * When `<= 0`, the whole base topology is reordered in a single in-memory pass (no temporary
+   * files). When set, the base topology is reordered through bucketed temporary files so that peak
+   * host memory stays close to this budget.
+   */
+  double max_host_memory_gb;
+  /** Number of host threads to use. When `0`, the maximum number of threads is used. */
+  int num_threads;
+};
+
+typedef struct cuvsHnswMaterializeParams* cuvsHnswMaterializeParams_t;
+
+/**
+ * @brief Allocate HNSW materialize params, and populate with default values
+ *
+ * @param[in] params cuvsHnswMaterializeParams_t to allocate
+ * @return cuvsError_t
+ */
+CUVS_EXPORT cuvsError_t cuvsHnswMaterializeParamsCreate(cuvsHnswMaterializeParams_t* params);
+
+/**
+ * @brief De-allocate HNSW materialize params
+ *
+ * @param[in] params cuvsHnswMaterializeParams_t to de-allocate
+ * @return cuvsError_t
+ */
+CUVS_EXPORT cuvsError_t cuvsHnswMaterializeParamsDestroy(cuvsHnswMaterializeParams_t params);
+
+/**
+ * @brief Materialize a layered HNSW artifact into a standard hnswlib index file on disk.
+ *
+ * Materializes a `GRAPH_ONLY` artifact (graph topology only, stored in ACE order) plus a
+ * local dataset into a standard hnswlib index file, without ever holding the full materialized
+ * index in host memory. The resulting file is compatible with the original hnswlib library and can
+ * be read back through `cuvsHnswDeserialize` with `hierarchy == CPU`. The element data type
+ * (`float`, `half`, `uint8_t` or `int8_t`) is inferred from the external dataset. GRAPH_ONLY
+ * artifacts are currently produced through the C++ API.
+ *
+ * @param[in] res cuvsResources_t opaque C handle
+ * @param[in] params cuvsHnswMaterializeParams_t materialization parameters
+ * @param[in] layered_artifact_path path to the layered HNSW artifact
+ * @param[in] output_path path to the hnswlib index file to write
+ * @param[in] dim the dimension of the vectors in the index
+ * @param[in] metric the distance metric used to build the index
+ * @return cuvsError_t
+ */
+CUVS_EXPORT cuvsError_t cuvsHnswMaterializeToHnswlib(cuvsResources_t res,
+                                                     cuvsHnswMaterializeParams_t params,
+                                                     const char* layered_artifact_path,
+                                                     const char* output_path,
+                                                     int dim,
+                                                     cuvsDistanceType metric);
+
+/**
+ * @}
+ */
+
 #ifdef __cplusplus
 }
 #endif

c/src/neighbors/hnsw.cpp

@@ -408,3 +408,41 @@ extern "C" cuvsError_t cuvsHnswDeserialize(cuvsResources_t res,
     }
   });
 }
+
+extern "C" cuvsError_t cuvsHnswMaterializeParamsCreate(cuvsHnswMaterializeParams_t* params)
+{
+  return cuvs::core::translate_exceptions([=] {
+    *params = new cuvsHnswMaterializeParams{
+      .dataset_path = nullptr, .max_host_memory_gb = 0, .num_threads = 0};
+  });
+}
+
+extern "C" cuvsError_t cuvsHnswMaterializeParamsDestroy(cuvsHnswMaterializeParams_t params)
+{
+  return cuvs::core::translate_exceptions([=] { delete params; });
+}
+
+extern "C" cuvsError_t cuvsHnswMaterializeToHnswlib(cuvsResources_t res,
+                                                    cuvsHnswMaterializeParams_t params,
+                                                    const char* layered_artifact_path,
+                                                    const char* output_path,
+                                                    int dim,
+                                                    cuvsDistanceType metric)
+{
+  return cuvs::core::translate_exceptions([=] {
+    auto res_ptr    = reinterpret_cast<raft::resources*>(res);
+    auto cpp_params = cuvs::neighbors::hnsw::materialize_params();
+    if (params->dataset_path != nullptr) {
+      cpp_params.dataset_path = std::string(params->dataset_path);
+    }
+    cpp_params.max_host_memory_gb = params->max_host_memory_gb;
+    cpp_params.num_threads        = params->num_threads;
+    auto metric_type              = static_cast<cuvs::distance::DistanceType>(metric);
+    cuvs::neighbors::hnsw::materialize_to_hnswlib(*res_ptr,
+                                                  cpp_params,
+                                                  std::string(layered_artifact_path),
+                                                  std::string(output_path),
+                                                  dim,
+                                                  metric_type);
+  });
+}

cpp/include/cuvs/neighbors/hnsw.hpp

@@ -1451,6 +1451,78 @@ void deserialize(raft::resources const& res,
  * @}
  */
 
+/**
+ * @defgroup hnsw_cpp_index_materialize Materialize a layered HNSW artifact into an hnswlib index
+ * @{
+ */
+
+/**
+ * @brief Parameters for materializing a layered HNSW artifact into an hnswlib index on disk.
+ */
+struct materialize_params {
+  /** Local dataset path holding the original-ID-ordered vectors used to build the artifact.
+   *
+   * Supported formats match layered deserialization: `.npy` and ANN benchmark `*.bin` files with a
+   * `[uint32 rows, uint32 cols]` header (`.fbin`, `.f16bin`, `.u8bin`, `.i8bin`).
+   */
+  std::string dataset_path;
+
+  /** Upper bound on host memory (in GiB) used for the base-topology reorder buffer.
+   *
+   * When `<= 0`, the whole base topology is reordered in a single in-memory pass (no temporary
+   * files). When set, the base topology is reordered through bucketed temporary files so that
+   * peak host memory stays close to this budget, at the cost of writing and re-reading the
+   * (small) base-topology section once.
+   */
+  double max_host_memory_gb = 0;
+
+  /** Number of host threads to use. When `0`, the maximum number of threads is used. */
+  int num_threads = 0;
+};
+
+/**
+ * @brief Materialize a layered HNSW artifact into a standard hnswlib index file on disk.
+ *
+ * Materializes a `GRAPH_ONLY` artifact (graph topology only, stored in ACE order) plus a
+ * local dataset into a standard hnswlib index file, without ever holding the full materialized
+ * index in host memory. The materialization reorders the base topology from ACE order to
+ * original-id order and interleaves the vectors, emitting the output with sequential disk IO. The
+ * resulting file is compatible with the original hnswlib library (`loadIndex`) and can be read back
+ * through `cuvs::neighbors::hnsw::deserialize` with `hierarchy == HnswHierarchy::CPU`.
+ *
+ * The element data type (`float`, `half`, `uint8_t` or `int8_t`) is inferred from the external
+ * dataset, so materialization supports an original dataset dtype that differs from the graph's
+ * construction dtype.
+ *
+ * @param[in] res raft resources
+ * @param[in] params materialization parameters (dataset path, host-memory budget, threads)
+ * @param[in] layered_artifact_path path to the layered HNSW artifact
+ * @param[in] output_path path to the hnswlib index file to write
+ * @param[in] dim dimensions of the training dataset
+ * @param[in] metric distance metric. Supported metrics ("L2Expanded", "InnerProduct")
+ *
+ * See the [layered HNSW
+ * example](https://github.com/NVIDIA/cuvs/blob/main/examples/cpp/src/hnsw_ace_layered_example.cu)
+ * for a complete materialization and search workflow.
+ * @code{.cpp}
+ *   using namespace cuvs::neighbors;
+ *   hnsw::materialize_params materialize_params;
+ *   materialize_params.dataset_path = "dataset.fbin";
+ *   hnsw::materialize_to_hnswlib(
+ *     res, materialize_params, "layered_artifact.cuvs", "index.bin", dim, metric);
+ * @endcode
+ */
+void materialize_to_hnswlib(raft::resources const& res,
+                            const materialize_params& params,
+                            const std::string& layered_artifact_path,
+                            const std::string& output_path,
+                            int dim,
+                            cuvs::distance::DistanceType metric);
+
+/**
+ * @}
+ */
+
 }  // namespace hnsw
 }  // namespace neighbors
 }  // namespace CUVS_EXPORT cuvs

cpp/include/cuvs/util/file_io.hpp

@@ -443,6 +443,19 @@ void write_large_file(const file_descriptor& fd,
                       const size_t total_bytes,
                       const uint64_t file_offset);
 
+/**
+ * @brief Pre-size a file to `total_bytes` bytes.
+ *
+ * Prefers posix_fallocate (reserves blocks up-front, avoids later ENOSPC and fragmentation), but
+ * falls back to ftruncate on filesystems that do not support preallocation (tmpfs and some
+ * NFS/overlay mounts return EOPNOTSUPP/EINVAL/ENOSYS) so the operation still succeeds there. A
+ * `total_bytes` of 0 is a no-op. Throws on failure (uses the descriptor's path in the message).
+ *
+ * @param fd File descriptor to pre-size
+ * @param total_bytes Target file size in bytes
+ */
+void preallocate_file(const file_descriptor& fd, const size_t total_bytes);
+
 /**
  * @brief Sequential std::ostream backed by kvikio.
  *