diff --git a/src/common/btree.c b/src/common/btree.c index 4ec39e2553b..06130bd0bca 100644 --- a/src/common/btree.c +++ b/src/common/btree.c @@ -170,7 +170,7 @@ struct btr_context { static int btr_class_init(umem_off_t root_off, struct btr_root *root, unsigned int tree_class, uint64_t *tree_feats, struct umem_attr *uma, daos_handle_t coh, void *priv, - btr_report_fn_t report_fn, void *report_arg, struct btr_instance *tins); + report_fn_t report_fn, void *report_arg, struct btr_instance *tins); static struct btr_record *btr_node_rec_at(struct btr_context *tcx, umem_off_t nd_off, unsigned int at); @@ -318,11 +318,6 @@ btr_ops(struct btr_context *tcx) return tcx->tc_tins.ti_ops; } -static inline void -report_fn_nop(void *arg, enum btr_report_type type, const char *fmt, ...) -{ -} - /** * Create a btree context (in volatile memory). * @@ -4596,7 +4591,7 @@ btr_class_feats_init(unsigned int tree_class, uint64_t *tree_feats, struct btr_c static int btr_class_init(umem_off_t root_off, struct btr_root *root, unsigned int tree_class, uint64_t *tree_feats, struct umem_attr *uma, daos_handle_t coh, void *priv, - btr_report_fn_t report_fn, void *report_arg, struct btr_instance *tins) + report_fn_t report_fn, void *report_arg, struct btr_instance *tins) { struct btr_class *tc; int rc; @@ -4624,29 +4619,28 @@ btr_class_init(umem_off_t root_off, struct btr_root *root, unsigned int tree_cla /* XXX should be multi-thread safe */ if (tree_class >= BTR_TYPE_MAX || DAOS_FAIL_CHECK(DAOS_FAULT_BTREE_OPEN_INV_CLASS)) { - report_fn(report_arg, BTR_REPORT_ERROR, TREE_CLASS_STR INVALID_CLASS_FMT, - tree_class); + report_fn(report_arg, REPORT_ERROR, TREE_CLASS_STR INVALID_CLASS_FMT, tree_class); D_DEBUG(DB_TRACE, INVALID_CLASS_FMT, tree_class); return -DER_INVAL; } tc = &btr_class_registered[tree_class]; if (tc->tc_ops == NULL || DAOS_FAIL_CHECK(DAOS_FAULT_BTREE_OPEN_UNREG_CLASS)) { - report_fn(report_arg, BTR_REPORT_ERROR, TREE_CLASS_STR UNREGISTERED_CLASS_FMT, + report_fn(report_arg, REPORT_ERROR, TREE_CLASS_STR UNREGISTERED_CLASS_FMT, tree_class); D_DEBUG(DB_TRACE, UNREGISTERED_CLASS_FMT, tree_class); return -DER_NONEXIST; } - report_fn(report_arg, BTR_REPORT_MSG, TREE_CLASS_STR OK_STR); + report_fn(report_arg, REPORT_MSG, TREE_CLASS_STR OK_STR); rc = btr_class_feats_init(tree_class, tree_feats, tc); if (rc != DER_SUCCESS) { - report_fn(report_arg, BTR_REPORT_ERROR, TREE_FEATURES_STR UNSUPPORTED_FEATURES_FMT, + report_fn(report_arg, REPORT_ERROR, TREE_FEATURES_STR UNSUPPORTED_FEATURES_FMT, *tree_feats, tc->tc_feats); D_ERROR(UNSUPPORTED_FEATURES_FMT, *tree_feats, tc->tc_feats); return rc; } - report_fn(report_arg, BTR_REPORT_MSG, TREE_FEATURES_STR OK_STR); + report_fn(report_arg, REPORT_MSG, TREE_FEATURES_STR OK_STR); tins->ti_ops = tc->tc_ops; return rc; @@ -4761,6 +4755,17 @@ dbtree_overhead_get(int alloc_overhead, unsigned int tclass, uint64_t otype, return 0; } +static int +btr_rec_check(struct btr_context *tcx, struct btr_record *rec, report_fn_t report_fn, + void *report_arg) +{ + if (!btr_ops(tcx)->to_rec_check) { + return -DER_NOSYS; + } + + return btr_ops(tcx)->to_rec_check(&tcx->tc_tins, rec, report_fn, report_arg); +} + #define CK_BTREE_NODE_FMT "Node (off=%#lx)... " #define CK_BTREE_NODE_MALFORMED_STR "malformed - " #define CK_BTREE_NON_ZERO_PADDING_FMT CK_BTREE_NODE_MALFORMED_STR "tn_pad_32 != 0 (%#" PRIx32 ")" @@ -4771,13 +4776,14 @@ dbtree_overhead_get(int alloc_overhead, unsigned int tclass, uint64_t otype, * * \param[in] nd Node to check. * \param[in] nd_off Node's offset. - * \param[in] ck Checker. + * \param[in] report_fn Report function. + * \param[in] report_arg Argument for the report function. * * \retval DER_SUCCESS The node is correct. * \retval -DER_NOTYPE The node is malformed. */ static int -btr_node_check(struct btr_node *nd, umem_off_t nd_off, btr_report_fn_t report_fn, void *report_arg, +btr_node_check(struct btr_node *nd, umem_off_t nd_off, report_fn_t report_fn, void *report_arg, bool error_on_non_zero_padding) { uint16_t unknown_flags; @@ -4786,7 +4792,7 @@ btr_node_check(struct btr_node *nd, umem_off_t nd_off, btr_report_fn_t report_fn unknown_flags = nd->tn_flags & ~(BTR_NODE_LEAF | BTR_NODE_ROOT); if (unknown_flags != 0) { - report_fn(report_arg, BTR_REPORT_ERROR, + report_fn(report_arg, REPORT_ERROR, CK_BTREE_NODE_MALFORMED_STR "unknown flags (%#" PRIx16 ")", unknown_flags); return -DER_NOTYPE; @@ -4794,12 +4800,12 @@ btr_node_check(struct btr_node *nd, umem_off_t nd_off, btr_report_fn_t report_fn if (nd->tn_pad_32 != 0) { if (error_on_non_zero_padding) { - report_fn(report_arg, BTR_REPORT_ERROR, + report_fn(report_arg, REPORT_ERROR, CK_BTREE_NODE_FMT CK_BTREE_NON_ZERO_PADDING_FMT, nd_off, nd->tn_pad_32); return -DER_NOTYPE; } else { - report_fn(report_arg, BTR_REPORT_WARNING, + report_fn(report_arg, REPORT_WARNING, CK_BTREE_NODE_FMT CK_BTREE_NON_ZERO_PADDING_FMT, nd_off, nd->tn_pad_32); } @@ -4807,16 +4813,16 @@ btr_node_check(struct btr_node *nd, umem_off_t nd_off, btr_report_fn_t report_fn if (nd->tn_gen != 0) { if (error_on_non_zero_padding) { - report_fn(report_arg, BTR_REPORT_ERROR, + report_fn(report_arg, REPORT_ERROR, CK_BTREE_NODE_FMT CK_BTREE_NON_ZERO_GEN_FMT, nd_off, nd->tn_gen); return -DER_NOTYPE; } else { - report_fn(report_arg, BTR_REPORT_WARNING, + report_fn(report_arg, REPORT_WARNING, CK_BTREE_NODE_FMT CK_BTREE_NON_ZERO_GEN_FMT, nd_off, nd->tn_gen); } } - report_fn(report_arg, BTR_REPORT_MSG, CK_BTREE_NODE_FMT OK_STR, nd_off); + report_fn(report_arg, REPORT_MSG, CK_BTREE_NODE_FMT OK_STR, nd_off); return DER_SUCCESS; } @@ -4835,7 +4841,8 @@ struct node_info { * Validate the integrity of a btree. * * \param[in] tcx Btree context. - * \param[in] ck Checker. + * \param[in] report_fn Report function. + * \param[in] report_arg Argument for the report function. * * \retval DER_SUCCESS The tree is correct. * \retval -DER_NOTYPE The tree is malformed. @@ -4843,7 +4850,7 @@ struct node_info { * \retval -DER_* Possibly other errors. */ static int -btr_nodes_check(struct btr_context *tcx, btr_report_fn_t report_fn, void *report_arg, +btr_nodes_check(struct btr_context *tcx, report_fn_t report_fn, void *report_arg, bool error_on_non_zero_padding) { D_LIST_HEAD(node_list); @@ -4851,12 +4858,13 @@ btr_nodes_check(struct btr_context *tcx, btr_report_fn_t report_fn, void *report struct node_info *ni_tmp; umem_off_t nd_off; struct btr_node *nd; + struct btr_record *rec; int rc = DER_SUCCESS; D_ASSERT(report_fn != NULL); if (btr_root_empty(tcx)) { - report_fn(report_arg, BTR_REPORT_MSG, "Empty tree\n"); + report_fn(report_arg, REPORT_MSG, "Empty tree\n"); return DER_SUCCESS; } @@ -4872,6 +4880,7 @@ btr_nodes_check(struct btr_context *tcx, btr_report_fn_t report_fn, void *report ni = d_list_pop_entry(&node_list, struct node_info, link); nd_off = ni->nd_off; nd = btr_off2ptr(tcx, nd_off); + D_FREE(ni); /** check the node */ rc = btr_node_check(nd, nd_off, report_fn, report_arg, error_on_non_zero_padding); @@ -4879,16 +4888,30 @@ btr_nodes_check(struct btr_context *tcx, btr_report_fn_t report_fn, void *report break; } + /** check records' consistency */ + report_fn(report_arg, REPORT_INDENT_INC, NULL); + for (int at = 0; at < nd->tn_keyn; ++at) { + rec = btr_node_rec_at(tcx, nd_off, at); + rc = btr_rec_check(tcx, rec, report_fn, report_arg); + if (rc != DER_SUCCESS) { + break; + } + } + report_fn(report_arg, REPORT_INDENT_DEC, NULL); + if (rc != DER_SUCCESS) { + break; + } + /** a leaf has no child nodes */ if (btr_node_is_leaf(tcx, nd_off)) { continue; } /** - * append the node's children to the front of the nodes' list + * Append the node's children to the front of the nodes' list. * - * Note: This makes the traversal depth-first. Given the limited depth of a typical - * DAOS tree, this approach should help reduce resource usage. + * Note: This makes the traversal depth-first. Given the limited depth of a + * typical DAOS tree, this approach should help reduce resource usage. */ for (int at = 0; at < nd->tn_keyn; ++at) { D_ALLOC_PTR(ni); @@ -4912,11 +4935,14 @@ btr_nodes_check(struct btr_context *tcx, btr_report_fn_t report_fn, void *report * * \param[in] root Address of the tree root. * \param[in] uma Memory class attributes. - * \param[in] ck Checker. + * \param[in] priv Private data for the tree class. + * \param[in] report_fn Report function. + * \param[in] report_arg Argument for the report function. + * \param[in] error_on_non_zero_padding Trigger an error on non-zero padding. */ int -dbtree_check_inplace(struct btr_root *root, struct umem_attr *uma, btr_report_fn_t report_fn, - void *report_arg, bool error_on_non_zero_padding) +dbtree_check_inplace(struct btr_root *root, struct umem_attr *uma, void *priv, + report_fn_t report_fn, void *report_arg, bool error_on_non_zero_padding) { struct btr_context tcx = {0}; uint64_t tree_feats = -1; @@ -4926,7 +4952,7 @@ dbtree_check_inplace(struct btr_root *root, struct umem_attr *uma, btr_report_fn D_ASSERT(uma != NULL); D_ASSERT(report_fn != NULL); - rc = btr_class_init(UMOFF_NULL, root, -1, &tree_feats, uma, DAOS_HDL_INVAL, NULL, report_fn, + rc = btr_class_init(UMOFF_NULL, root, -1, &tree_feats, uma, DAOS_HDL_INVAL, priv, report_fn, report_arg, &tcx.tc_tins); if (rc != DER_SUCCESS) { return rc; diff --git a/src/include/daos/btree.h b/src/include/daos/btree.h index 9b9b243d9bb..b0d11c17dd0 100644 --- a/src/include/daos/btree.h +++ b/src/include/daos/btree.h @@ -1,6 +1,6 @@ /** * (C) Copyright 2016-2024 Intel Corporation. - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -16,6 +16,7 @@ #include #include #include +#include /** * KV record of the btree. @@ -409,7 +410,21 @@ typedef struct { * \a return Allocated node address (offset within the pool) */ umem_off_t (*to_node_alloc)(struct btr_instance *tins, int size); - + /** + * Optional: + * Check the consistency of the given record. + * + * \param tins [IN] Tree instance which contains the root umem + * offset and memory class etc. + * \param rec [IN] Record to be checked. + * \param report_fn [IN] Report function. + * \param report_arg [IN] Argument for the report function. + * + * \retval DER_SUCCESS Success. + * \retval -DER_* Errors returned by the fetch callback. + */ + int (*to_rec_check)(struct btr_instance *tins, struct btr_record *rec, + report_fn_t report_fn, void *report_arg); } btr_ops_t; /** @@ -541,17 +556,12 @@ int dbtree_open(umem_off_t root_off, struct umem_attr *uma, daos_handle_t *toh); int dbtree_open_inplace(struct btr_root *root, struct umem_attr *uma, daos_handle_t *toh); -int dbtree_open_inplace_ex(struct btr_root *root, struct umem_attr *uma, - daos_handle_t coh, void *priv, daos_handle_t *toh); -enum btr_report_type { - BTR_REPORT_ERROR, - BTR_REPORT_WARNING, - BTR_REPORT_MSG, -}; -typedef void (*btr_report_fn_t)(void *arg, enum btr_report_type type, const char *fmt, ...); int - dbtree_check_inplace(struct btr_root *root, struct umem_attr *uma, btr_report_fn_t report_fn, - void *report_arg, bool error_on_non_zero_padding); +dbtree_open_inplace_ex(struct btr_root *root, struct umem_attr *uma, daos_handle_t coh, void *priv, + daos_handle_t *toh); +int + dbtree_check_inplace(struct btr_root *root, struct umem_attr *uma, void *priv, + report_fn_t report_fn, void *report_arg, bool error_on_non_zero_padding); int dbtree_close(daos_handle_t toh); int dbtree_destroy(daos_handle_t toh, void *args); int dbtree_drain(daos_handle_t toh, int *credits, void *args, bool *destroyed); diff --git a/src/include/daos/common.h b/src/include/daos/common.h index 7a902466fa7..1aa4ecb180a 100644 --- a/src/include/daos/common.h +++ b/src/include/daos/common.h @@ -972,6 +972,18 @@ enum { #define DAOS_FAULT_POOL_EXT_PADDING (DAOS_FAIL_SYS_TEST_GROUP_LOC | 0x209) #define DAOS_FAULT_POOL_EXT_RESERVED (DAOS_FAIL_SYS_TEST_GROUP_LOC | 0x20a) +/** Container open fault injection */ +#define DAOS_FAULT_CONT_DOES_NOT_EXIST (DAOS_FAIL_SYS_TEST_GROUP_LOC | 0x300) +#define DAOS_FAULT_CONT_OPEN_UUID (DAOS_FAIL_SYS_TEST_GROUP_LOC | 0x301) +#define DAOS_FAULT_CONT_INV_PAD (DAOS_FAIL_SYS_TEST_GROUP_LOC | 0x302) +#define DAOS_FAULT_CONT_INV_USED (DAOS_FAIL_SYS_TEST_GROUP_LOC | 0x303) +#define DAOS_FAULT_CONT_INV_RESERV_UPGRADE (DAOS_FAIL_SYS_TEST_GROUP_LOC | 0x304) +#define DAOS_FAULT_DBD_MAGIC (DAOS_FAIL_SYS_TEST_GROUP_LOC | 0x305) +#define DAOS_FAULT_DAE_INV_FLAGS (DAOS_FAIL_SYS_TEST_GROUP_LOC | 0x306) +#define DAOS_FAULT_DAE_ALLOC (DAOS_FAIL_SYS_TEST_GROUP_LOC | 0x307) +#define DAOS_FAULT_DBD_COUNT (DAOS_FAIL_SYS_TEST_GROUP_LOC | 0x308) +#define DAOS_FAULT_OBJ_ILOG_MAGIC (DAOS_FAIL_SYS_TEST_GROUP_LOC | 0x309) + #define DAOS_DTX_SKIP_PREPARE DAOS_DTX_SPEC_LEADER #define DAOS_FAIL_CHECK(id) daos_fail_check(id) diff --git a/src/include/daos/report.h b/src/include/daos/report.h new file mode 100644 index 00000000000..a4fada422b7 --- /dev/null +++ b/src/include/daos/report.h @@ -0,0 +1,28 @@ +/** + * (C) Copyright 2026 Hewlett Packard Enterprise Development LP + * + * SPDX-License-Identifier: BSD-2-Clause-Patent + */ + +#ifndef __DAOS_REPORT_H__ +#define __DAOS_REPORT_H__ + +enum report_opts { + REPORT_ERROR, + REPORT_WARNING, + REPORT_MSG, + /** flags occupy the highest bits */ + REPORT_NO_PREFIX = (1 << 29), + REPORT_INDENT_INC = (1 << 30), + REPORT_INDENT_DEC = (1 << 31), + REPORT_FLAGS_MASK = (REPORT_NO_PREFIX | REPORT_INDENT_INC | REPORT_INDENT_DEC), +}; + +typedef void (*report_fn_t)(void *arg, enum report_opts opts, const char *fmt, ...); + +static inline void +report_fn_nop(void *arg, enum report_opts ops, const char *fmt, ...) +{ +} + +#endif /* __DAOS_REPORT_H__ */ diff --git a/src/include/daos_srv/checker.h b/src/include/daos_srv/checker.h index c6e03de3726..0af6dea0119 100644 --- a/src/include/daos_srv/checker.h +++ b/src/include/daos_srv/checker.h @@ -1,5 +1,5 @@ /** - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -85,34 +85,51 @@ ck_common_printf(struct checker *ck, const char *fmt, ...) * Print a btree report as a checker message. * * \param[in] arg Checker. - * \param[in] type Btree report type. + * \param[in] opts Btree report options. * \param[in] fmt Format. * \param[in] ... Format's arguments. */ static inline void -ck_report(void *arg, enum btr_report_type type, const char *fmt, ...) +ck_report(void *arg, enum report_opts opts, const char *fmt, ...) { - struct checker *ck = arg; + struct checker *ck = arg; + const char *prefix = (opts & REPORT_NO_PREFIX) ? "" : ck->ck_prefix; va_list args; + if (fmt == NULL) { + switch (opts) { + case REPORT_INDENT_INC: + ck->ck_level++; + ck->ck_indent_set(ck); + return; + case REPORT_INDENT_DEC: + ck->ck_level--; + ck->ck_indent_set(ck); + return; + default: + D_ASSERTF(0, "Unknown report options: %x\n", opts); + } + return; + } + va_start(args, fmt); - switch (type) { - case BTR_REPORT_ERROR: - ck_common_printf(ck, "%s%s", ck->ck_prefix, CHECKER_ERROR_INFIX); + switch (opts & ~REPORT_FLAGS_MASK) { + case REPORT_ERROR: + ck_common_printf(ck, "%s%s", prefix, CHECKER_ERROR_INFIX); ck->ck_vprintf(ck, fmt, args); break; - case BTR_REPORT_WARNING: - ck_common_printf(ck, "%s%s", ck->ck_prefix, CHECKER_WARNING_INFIX); - ck_common_printf(ck, fmt, args); + case REPORT_WARNING: + ck_common_printf(ck, "%s%s", prefix, CHECKER_WARNING_INFIX); + ck->ck_vprintf(ck, fmt, args); ck->ck_warnings_num++; break; - case BTR_REPORT_MSG: - ck_common_printf(ck, "%s", ck->ck_prefix); - ck_common_printf(ck, fmt, args); + case REPORT_MSG: + ck_common_printf(ck, "%s", prefix); + ck->ck_vprintf(ck, fmt, args); break; default: - D_ASSERTF(0, "Unknown report type: %x\n", type); + D_ASSERTF(0, "Unknown report type: %x\n", opts); } va_end(args); @@ -178,6 +195,14 @@ ck_report(void *arg, enum btr_report_type type, const char *fmt, ...) ++(ck)->ck_warnings_num; \ } while (0) +#define CK_APPENDL_WARN(ck, msg) \ + do { \ + CK_PRINT_WO_PREFIX(ck, CHECKER_WARNING_INFIX msg "\n"); \ + ++(ck)->ck_warnings_num; \ + } while (0) + +#define CK_APPENDL(ck, msg) CK_PRINT_WO_PREFIX(ck, msg "\n") + /** print(f) + return code + new line shortcuts */ #define CK_PRINTL_RC(ck, rc, msg) \ diff --git a/src/include/daos_srv/vos.h b/src/include/daos_srv/vos.h index 41bf04ad749..7cd8e310f40 100644 --- a/src/include/daos_srv/vos.h +++ b/src/include/daos_srv/vos.h @@ -583,6 +583,19 @@ vos_cont_destroy(daos_handle_t poh, uuid_t co_uuid); int vos_cont_open(daos_handle_t poh, uuid_t co_uuid, daos_handle_t *coh); +/** + * Open a container within a VOSP with a checker. + * + * \param poh [IN] Pool open handle + * \param co_uuid [IN] Container uuid + * \param ck [IN] Checker structure + * \param coh [OUT] Returned container handle + * + * \return Zero on success, negative value if error + */ +int +vos_cont_open_ex(daos_handle_t poh, uuid_t co_uuid, struct checker *ck, daos_handle_t *coh); + /** * Release container open handle * @@ -1332,6 +1345,16 @@ vos_iter_empty(daos_handle_t ih); int vos_iter_validate(daos_handle_t ih); +/** + * XXX + * + * \param[in] ih XXX + * + * \retval XXX + */ +int +vos_iter_check(daos_handle_t ih, vos_iter_entry_t *entry, vos_iter_type_t type, struct checker *ck); + /** * Iterate VOS entries (i.e., containers, objects, dkeys, etc.) and call \a * cb(\a arg) for each entry. diff --git a/src/include/daos_srv/vos_types.h b/src/include/daos_srv/vos_types.h index 696cf7f3945..9f51baaac8f 100644 --- a/src/include/daos_srv/vos_types.h +++ b/src/include/daos_srv/vos_types.h @@ -63,6 +63,10 @@ enum dtx_entry_flags { DTE_EPOCH_SORTED = (1 << 6), }; +#define DTE_FLAGS_VALID \ + (DTE_LEADER | DTE_INVALID | DTE_BLOCK | DTE_CORRUPTED | DTE_ORPHAN | \ + DTE_PARTIAL_COMMITTED | DTE_EPOCH_SORTED) + struct dtx_entry { /** The identifier of the DTX. */ struct dtx_id dte_xid; diff --git a/src/utils/dlck/cmds/dlck_cmd_check.c b/src/utils/dlck/cmds/dlck_cmd_check.c index b48116b6dd6..0a0992c8810 100644 --- a/src/utils/dlck/cmds/dlck_cmd_check.c +++ b/src/utils/dlck/cmds/dlck_cmd_check.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -15,6 +15,129 @@ #include "../dlck_pool.h" #include "../dlck_report.h" +#define DLCK_CHECK_RESULT_PREFIX_FMT(TYPE_STR) "[%d] " TYPE_STR " " DF_UUIDF " check result" +#define DLCK_WARNINGS_NUM_FMT " (%u warning(s))" + +#define REPORT_RESULT(CK, PREFIX_FMT, TGT_ID, UUID, RC, WARN_NUM) \ + do { \ + if (RC == DER_SUCCESS && WARN_NUM > 0) { \ + CK_PRINTF(CK, PREFIX_FMT CHECKER_OK_INFIX DLCK_WARNINGS_NUM_FMT ".\n", \ + TGT_ID, DP_UUID(UUID), WARN_NUM); \ + } else { \ + CK_PRINTFL_RC(CK, RC, PREFIX_FMT, TGT_ID, DP_UUID(UUID)); \ + } \ + } while (0) + +#define POOL_REPORT_RESULT(CK, TGT_ID, UUID, RC, WARN_NUM) \ + REPORT_RESULT(CK, DLCK_CHECK_RESULT_PREFIX_FMT("pool "), TGT_ID, UUID, RC, WARN_NUM) + +#define CONT_REPORT_RESULT(CK, TGT_ID, UUID, RC, WARN_NUM) \ + REPORT_RESULT(CK, DLCK_CHECK_RESULT_PREFIX_FMT("container"), TGT_ID, UUID, RC, WARN_NUM) + +struct bundle { + struct xstream_arg *xa; + struct checker *ck; +}; + +static int +obj_process(daos_handle_t ih, vos_iter_entry_t *entry, vos_iter_type_t type, + vos_iter_param_t *param, void *cb_arg, unsigned int *acts) +{ + struct bundle *bndl = cb_arg; + // struct xstream_arg *xa = bndl->xa; + // struct checker *main_ck = &xa->ctrl->checker; + struct checker *ck = bndl->ck; + int rc; + + CK_PRINTF(ck, "oid: " DF_UOID "\n", DP_UOID(entry->ie_oid)); + + rc = vos_iter_check(ih, entry, type, ck); + + return 0; +} + +/** + * Target thread (worker). Check trees of a single container. + * + * \param[in] ck Checker. + * \param[in] cont Container to check. + * + * \retval DER_SUCCESS Success. + * \retval -DER_* Errors returned by the tree checking logic. + */ +static int +trees_process(daos_handle_t coh, struct bundle *bndl) +{ + vos_iter_param_t param = {0}; + struct vos_iter_anchors anchors = {0}; + + param.ip_hdl = coh; + param.ip_epr.epr_hi = DAOS_EPOCH_MAX; + param.ip_flags = VOS_IT_FOR_CHECK; + + return vos_iterate(¶m, VOS_ITER_OBJ, false, &anchors, obj_process, NULL, bndl, NULL); +} + +/** + * Target thread (worker). VOS iterator callback. Check a single container. + * + * \param[in] ih Iterator handle. + * \param[in] entry Iterator entry. + * \param[in] type Iteration type. + * \param[in] param Iterator parameters. + * \param[in] cb_arg Callback argument. + * \param[in] acts Actions. + * + * \retval DER_SUCCESS Success. + * \retval -DER_* Errors returned by vos_cont_open_ex(). + */ +static int +cont_process(daos_handle_t ih, vos_iter_entry_t *entry, vos_iter_type_t type, + vos_iter_param_t *param, void *cb_arg, unsigned int *acts) +{ + struct bundle *bndl = cb_arg; + struct xstream_arg *xa = bndl->xa; + struct checker *main_ck = &xa->ctrl->checker; + struct checker *ck = bndl->ck; + daos_handle_t coh; + int rc; + + rc = vos_cont_open_ex(param->ip_hdl, entry->ie_couuid, ck, &coh); + CONT_REPORT_RESULT(main_ck, xa->xs->tgt_id, entry->ie_couuid, rc, ck->ck_warnings_num); + if (rc == DER_SUCCESS) { + trees_process(coh, bndl); + + (void)vos_cont_close(coh); + } + + /** continue checking other containers even if this one failed */ + return 0; +} + +/** + * Target thread (worker). Check all containers of a single pool. + * + * \param[in] poh Pool handle. + * \param[in] ck Checker. + * + * \retval DER_SUCCESS Success. + * \retval -DER_* Errors either from the VOS iterator or vos_cont_open_ex(). + */ +static int +conts_process(struct xstream_arg *xa, daos_handle_t poh, struct checker *ck) +{ + vos_iter_param_t param = {0}; + struct vos_iter_anchors anchors = {0}; + struct bundle cb_arg = {.xa = xa, .ck = ck}; + + param.ip_hdl = poh; + param.ip_epr.epr_hi = DAOS_EPOCH_MAX; + param.ip_flags = VOS_IT_FOR_CHECK; + + return vos_iterate(¶m, VOS_ITER_COUUID, false, &anchors, cont_process, NULL, &cb_arg, + NULL); +} + /** * Target thread (worker). Check a single pool. * @@ -29,6 +152,7 @@ static int pool_process(struct xstream_arg *xa, struct dlck_file *file, struct checker *ck) { + struct checker *main_ck = &xa->ctrl->checker; char *path; daos_handle_t poh; int rc; @@ -50,11 +174,14 @@ pool_process(struct xstream_arg *xa, struct dlck_file *file, struct checker *ck) rc = vos_pool_open_metrics(path, file->po_uuid, DLCK_POOL_OPEN_FLAGS, NULL, ck, &poh); if (rc == DER_SUCCESS) { + POOL_REPORT_RESULT(main_ck, xa->xs->tgt_id, file->po_uuid, rc, ck->ck_warnings_num); + rc = conts_process(xa, poh, ck); + (void)vos_pool_close(poh); } D_FREE(path); - /** check */ + /** check */ if (rc != DER_SUCCESS) { /** ignore a possible error from the unlock */ return rc; @@ -63,9 +190,6 @@ pool_process(struct xstream_arg *xa, struct dlck_file *file, struct checker *ck) return DER_SUCCESS; } -#define DLCK_POOL_CHECK_RESULT_PREFIX_FMT "[%d] pool " DF_UUIDF " check result" -#define DLCK_WARNINGS_NUM_FMT " (%u warning(s))" - /** * Target thread (worker). */ @@ -106,17 +230,6 @@ exec_one(void *arg) /** check the pool */ rc = pool_process(xa, file, &ck); - /** report the result */ - if (rc == DER_SUCCESS && ck.ck_warnings_num > 0) { - CK_PRINTF( - main_ck, - DLCK_POOL_CHECK_RESULT_PREFIX_FMT CHECKER_OK_INFIX DLCK_WARNINGS_NUM_FMT - ".\n", - xa->xs->tgt_id, DP_UUID(file->po_uuid), ck.ck_warnings_num); - } else { - CK_PRINTFL_RC(main_ck, rc, DLCK_POOL_CHECK_RESULT_PREFIX_FMT, - xa->xs->tgt_id, DP_UUID(file->po_uuid)); - } dlck_xstream_set_rc(xa, rc); dlck_uadd_no_overflow(xa->warnings_num, ck.ck_warnings_num, &xa->warnings_num); /** Continue to the next pool regardless of the result. */ diff --git a/src/utils/dlck/dlck_engine.c b/src/utils/dlck/dlck_engine.c index 46670f3a443..6088df128ec 100644 --- a/src/utils/dlck/dlck_engine.c +++ b/src/utils/dlck/dlck_engine.c @@ -1,5 +1,5 @@ /** - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -700,7 +700,7 @@ dlck_engine_exec_all(struct dlck_engine *engine, dlck_ult_func exec_one, CK_PRINT(ck, "Start targets... "); rc = dlck_engine_targets_start(engine, exec_one, arg_alloc_fn, &de); - CK_APPENDL_OK(ck); + CK_APPENDL_RC(ck, rc); if (rc != DER_SUCCESS) { return rc; } diff --git a/src/utils/dlck/dlck_main.c b/src/utils/dlck/dlck_main.c index d380de67c60..dafb146fd6e 100644 --- a/src/utils/dlck/dlck_main.c +++ b/src/utils/dlck/dlck_main.c @@ -94,7 +94,7 @@ user_belongs_to_group(const char *group_name, struct checker *ck) } if (rc < 0) { rc = daos_errno2der(errno); - CK_PRINTFL_RC(ck, rc, "getgroups() failed", group_name); + CK_PRINTL_RC(ck, rc, "getgroups() failed"); return false; } diff --git a/src/utils/dlck/tests/fault_injection_dlck.yaml b/src/utils/dlck/tests/fault_injection_dlck.yaml index 5d822eaaa02..fb4553c02ab 100644 --- a/src/utils/dlck/tests/fault_injection_dlck.yaml +++ b/src/utils/dlck/tests/fault_injection_dlck.yaml @@ -36,3 +36,14 @@ fault_config: # max_faults: 1 # - id: 131593 # DAOS_FAULT_POOL_EXT_PADDING # - id: 131594 # DAOS_FAULT_POOL_EXT_RESERVED + # - id: 131840 # DAOS_FAULT_CONT_DOES_NOT_EXIST + # interval: 2 # skip sys_db + # - id: 131841 # DAOS_FAULT_CONT_OPEN_UUID + # - id: 131842 # DAOS_FAULT_CONT_INV_PAD + # - id: 131843 # DAOS_FAULT_CONT_INV_USED + # - id: 131844 # DAOS_FAULT_CONT_INV_RESERV_UPGRADE + # - id: 131845 # DAOS_FAULT_DBD_MAGIC + # - id: 131846 # DAOS_FAULT_DAE_INV_FLAGS + # - id: 131847 # DAOS_FAULT_DAE_ALLOC + # - id: 131848 # DAOS_FAULT_DBD_COUNT + # - id: 131849 # DAOS_FAULT_OBJ_ILOG_MAGIC diff --git a/src/vos/ilog.c b/src/vos/ilog.c index 016cf8fa86f..25ca2898d76 100644 --- a/src/vos/ilog.c +++ b/src/vos/ilog.c @@ -1,6 +1,6 @@ /** * (C) Copyright 2019-2024 Intel Corporation. - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP. + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP. * (C) Copyright 2025 Google LLC * * SPDX-License-Identifier: BSD-2-Clause-Patent @@ -1648,13 +1648,21 @@ ilog_version_get(daos_handle_t loh) return ilog_mag2ver(lctx->ic_root->lr_magic); } -bool -ilog_root_is_valid(struct ilog_df *ilog_df) +int +ilog_root_is_valid(struct ilog_df *ilog_df, report_fn_t report_fn, void *report_arg) { struct ilog_root *root = (struct ilog_root *)ilog_df; D_ASSERT(root != NULL); - return ILOG_MAGIC_VALID(root->lr_magic); + report_fn(report_arg, REPORT_MSG, "ILOG... "); + if (!ILOG_MAGIC_VALID(root->lr_magic) || DAOS_FAIL_CHECK(DAOS_FAULT_OBJ_ILOG_MAGIC)) { + report_fn(report_arg, REPORT_ERROR | REPORT_NO_PREFIX, + "invalid magic (%#" PRIx32 ").\n", root->lr_magic); + return -DER_DF_INVAL; + } + report_fn(report_arg, REPORT_MSG | REPORT_NO_PREFIX, CHECKER_OK_INFIX ".\n"); + + return DER_SUCCESS; } bool diff --git a/src/vos/ilog.h b/src/vos/ilog.h index 3fdff8524a5..a14ab9d256e 100644 --- a/src/vos/ilog.h +++ b/src/vos/ilog.h @@ -1,6 +1,6 @@ /** * (C) Copyright 2019-2024 Intel Corporation. - * (C) Copyright 2025 Hewlett Packard Enterprise Development LP. + * (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -344,11 +344,14 @@ ilog_is_punch(const struct ilog_entry *entry) * Validate ilog's root. * * \param[in] ilog_df + * \param[in] report_fn Report function. + * \param[in] report_arg Argument for the report function. * - * \return true if the root is valid. + * \retval DER_SUCCESS On success. + * \retval -DER_DF_INVAL Invalid ilog magic. */ -bool -ilog_root_is_valid(struct ilog_df *ilog_df); +int +ilog_root_is_valid(struct ilog_df *ilog_df, report_fn_t report_fn, void *report_arg); /** Validate the provided ilog. * diff --git a/src/vos/vos_container.c b/src/vos/vos_container.c index 388a065d605..be91a642c45 100644 --- a/src/vos/vos_container.c +++ b/src/vos/vos_container.c @@ -147,14 +147,23 @@ cont_df_rec_update(struct btr_instance *tins, struct btr_record *rec, return 0; } +static int +cont_df_rec_check(struct btr_instance *tins, struct btr_record *rec, report_fn_t report_fn, + void *report_arg) +{ + /** NOP. The container is checked on open. */ + return 0; +} + static btr_ops_t vct_ops = { - .to_rec_msize = cont_df_rec_msize, - .to_hkey_size = cont_df_hkey_size, - .to_hkey_gen = cont_df_hkey_gen, - .to_rec_alloc = cont_df_rec_alloc, - .to_rec_free = cont_df_rec_free, - .to_rec_fetch = cont_df_rec_fetch, - .to_rec_update = cont_df_rec_update, + .to_rec_msize = cont_df_rec_msize, + .to_hkey_size = cont_df_hkey_size, + .to_hkey_gen = cont_df_hkey_gen, + .to_rec_alloc = cont_df_rec_alloc, + .to_rec_free = cont_df_rec_free, + .to_rec_fetch = cont_df_rec_fetch, + .to_rec_update = cont_df_rec_update, + .to_rec_check = cont_df_rec_check, }; static int @@ -337,13 +346,16 @@ static const struct lru_callbacks lru_cont_cbs = { .lru_on_free = vos_lru_free_track, }; +#define CK_NON_ZERO_FMT(PRIXX) "non-zero (%#" PRIXX ")" +#define CK_OBJ_TREE_STR "Object index tree" +#define CK_ACT_DBD_LIST_STR "Active DTX blob list" + /** - * Open a container within a VOSP + * Open a container within a VOSP with a checker. */ int -vos_cont_open(daos_handle_t poh, uuid_t co_uuid, daos_handle_t *coh) +vos_cont_open_ex(daos_handle_t poh, uuid_t co_uuid, struct checker *ck, daos_handle_t *coh) { - int rc = 0; struct vos_pool *pool = NULL; struct d_uuid ukey; @@ -352,6 +364,10 @@ vos_cont_open(daos_handle_t poh, uuid_t co_uuid, daos_handle_t *coh) struct vos_container *cont = NULL; struct umem_attr uma; + /** header with parameters */ + CK_PRINTF(ck, "Check container:\n\tuuid: " DF_UUIDF "\n", DP_UUID(co_uuid)); + checker_print_indent_inc(ck); + D_DEBUG(DB_TRACE, "Open container "DF_UUID"\n", DP_UUID(co_uuid)); pool = vos_hdl2pool(poh); @@ -368,6 +384,7 @@ vos_cont_open(daos_handle_t poh, uuid_t co_uuid, daos_handle_t *coh) */ rc = cont_lookup(&ukey, &pkey, &cont, pool->vp_sysdb); if (rc == 0) { + CK_PRINT(ck, "Container is already opened.\n"); cont->vc_open_count++; D_DEBUG(DB_TRACE, "Found handle for cont "DF_UUID " in DRAM hash table, open count: %d\n", @@ -376,13 +393,81 @@ vos_cont_open(daos_handle_t poh, uuid_t co_uuid, daos_handle_t *coh) D_GOTO(exit, rc); } - rc = cont_df_lookup(pool, &ukey, &args); + if (!DAOS_FAIL_CHECK(DAOS_FAULT_CONT_DOES_NOT_EXIST)) { + rc = cont_df_lookup(pool, &ukey, &args); + } else { + rc = daos_errno2der(daos_fail_value_get()); + } if (rc) { + CK_PRINT(ck, "Container does not exist.\n"); D_DEBUG(DB_TRACE, DF_UUID" container does not exist\n", DP_UUID(co_uuid)); D_GOTO(exit, rc); } + if (IS_CHECKER(ck)) { + CK_PRINT(ck, "uuid... "); + if (uuid_compare(args.ca_cont_df->cd_id, co_uuid) != 0 || + DAOS_FAIL_CHECK(DAOS_FAULT_CONT_OPEN_UUID)) { + CK_APPENDFL_ERR(ck, "mismatch (" DF_UUIDF " != " DF_UUIDF ")\n", + DP_UUID(args.ca_cont_df->cd_id), DP_UUID(co_uuid)); + D_GOTO(exit, rc = -DER_ID_MISMATCH); + } + CK_APPENDL_OK(ck); + + const bool error_on_non_zero_padding = + (ck->ck_options.cko_non_zero_padding == CHECKER_EVENT_ERROR); + + CK_PRINT(ck, "Padding (cd_pad)... "); + if (args.ca_cont_df->cd_pad != 0 || DAOS_FAIL_CHECK(DAOS_FAULT_CONT_INV_PAD)) { + if (error_on_non_zero_padding) { + CK_APPENDFL_ERR(ck, CK_NON_ZERO_FMT(PRIx32), + args.ca_cont_df->cd_pad); + D_GOTO(exit, rc = -DER_NOTYPE); + } else { + CK_APPENDFL_WARN(ck, CK_NON_ZERO_FMT(PRIx32), + args.ca_cont_df->cd_pad); + } + } + CK_APPENDL_OK(ck); + + CK_PRINT(ck, "Padding (cd_used)... "); + if (args.ca_cont_df->cd_used != 0 || DAOS_FAIL_CHECK(DAOS_FAULT_CONT_INV_USED)) { + if (error_on_non_zero_padding) { + CK_APPENDFL_ERR(ck, CK_NON_ZERO_FMT(PRIx64), + args.ca_cont_df->cd_used); + D_GOTO(exit, rc = -DER_NOTYPE); + } else { + CK_APPENDFL_WARN(ck, CK_NON_ZERO_FMT(PRIx64), + args.ca_cont_df->cd_used); + } + } + CK_APPENDL_OK(ck); + + CK_PRINT(ck, "Reserved (cd_reserv_upgrade)... "); + if (args.ca_cont_df->cd_reserv_upgrade != 0 || + DAOS_FAIL_CHECK(DAOS_FAULT_CONT_INV_RESERV_UPGRADE)) { + if (error_on_non_zero_padding) { + CK_APPENDFL_ERR(ck, CK_NON_ZERO_FMT(PRIx64), + args.ca_cont_df->cd_reserv_upgrade); + D_GOTO(exit, rc = -DER_NOTYPE); + } else { + CK_APPENDFL_WARN(ck, CK_NON_ZERO_FMT(PRIx64), + args.ca_cont_df->cd_reserv_upgrade); + } + } + CK_APPENDL_OK(ck); + + CK_PRINT(ck, CK_OBJ_TREE_STR "...\n"); + CK_INDENT(ck, rc = dbtree_check_inplace(&args.ca_cont_df->cd_obj_root, + &pool->vp_uma, pool, ck_report, ck, + error_on_non_zero_padding)); + CK_PRINTL_RC(ck, rc, CK_OBJ_TREE_STR); + if (rc != DER_SUCCESS) { + D_GOTO(exit, rc); + } + } + D_ALLOC_PTR(cont); if (!cont) { D_GOTO(exit, rc = -DER_NOMEM); @@ -484,7 +569,9 @@ vos_cont_open(daos_handle_t poh, uuid_t co_uuid, daos_handle_t *coh) */ cont->vc_mod_epoch_bound = d_hlc_get(); - rc = vos_dtx_act_reindex(cont); + CK_PRINT(ck, CK_ACT_DBD_LIST_STR "...\n"); + CK_INDENT(ck, rc = vos_dtx_act_reindex(cont, ck)); + CK_PRINTL_RC(ck, rc, CK_ACT_DBD_LIST_STR); if (rc != 0) { D_ERROR("Fail to reindex active DTX entries: %d\n", rc); goto exit; @@ -506,9 +593,21 @@ vos_cont_open(daos_handle_t poh, uuid_t co_uuid, daos_handle_t *coh) if (rc != 0 && cont) cont_free_internal(cont); + checker_print_indent_dec(ck); + CK_PRINTL_RC(ck, rc, "Check container"); + return rc; } +/** + * Open a container within a VOSP + */ +int +vos_cont_open(daos_handle_t poh, uuid_t co_uuid, daos_handle_t *coh) +{ + return vos_cont_open_ex(poh, co_uuid, NULL, coh); +} + /** * Release container open handle */ diff --git a/src/vos/vos_dtx.c b/src/vos/vos_dtx.c index 2795393f6ed..fd5691543c4 100644 --- a/src/vos/vos_dtx.c +++ b/src/vos/vos_dtx.c @@ -3600,8 +3600,10 @@ vos_dtx_mark_sync(daos_handle_t coh, daos_unit_oid_t oid, daos_epoch_t epoch) return 0; } +#define CK_DBD_FMT "DTX blob (off=%#lx)... " + int -vos_dtx_act_reindex(struct vos_container *cont) +vos_dtx_act_reindex(struct vos_container *cont, struct checker *ck) { struct umem_instance *umm = vos_cont2umm(cont); struct vos_cont_df *cont_df = cont->vc_cont_df; @@ -3617,41 +3619,100 @@ vos_dtx_act_reindex(struct vos_container *cont) /* The largest diff for above pairs 'max_eph - min_eph'. */ uint64_t diff = 0; uint64_t start_time = daos_wallclock_secs(); + int ck_level_cache = IS_CHECKER(ck) ? ck->ck_level : 0; int rc = 0; int i; + if (IS_CHECKER(ck) && UMOFF_IS_NULL(dbd_off)) { + CK_PRINT(ck, "No DTX blobs.\n"); + } + while (!UMOFF_IS_NULL(dbd_off)) { int dbd_count = 0; + CK_PRINTF(ck, CK_DBD_FMT "\n", dbd_off); + checker_print_indent_inc(ck); + dbd = umem_off2ptr(umm, dbd_off); - D_ASSERT(dbd->dbd_magic == DTX_ACT_BLOB_MAGIC); + + if (IS_NOT_CHECKER(ck)) { + D_ASSERT(dbd->dbd_magic == DTX_ACT_BLOB_MAGIC); + } else { + CK_PRINT(ck, "Magic... "); + if (dbd->dbd_magic != DTX_ACT_BLOB_MAGIC || + DAOS_FAIL_CHECK(DAOS_FAULT_DBD_MAGIC)) { + CK_APPENDFL_ERR(ck, "invalid (%#x)", dbd->dbd_magic); + D_GOTO(out, rc = -DER_DF_INVAL); + } + CK_APPENDL_OK(ck); + } + + if (IS_CHECKER(ck)) { + CK_PRINTF(ck, "%d active entries", dbd->dbd_index); + if (dbd->dbd_index > 0) { + CK_APPENDL(ck, ":"); + checker_print_indent_inc(ck); + } else { + CK_APPENDL(ck, "."); + } + } for (i = 0; i < dbd->dbd_index; i++) { struct vos_dtx_act_ent_df *dae_df; - struct vos_dtx_act_ent *dae; + struct vos_dtx_act_ent *dae = NULL; dae_df = &dbd->dbd_active_data[i]; - if (dae_df->dae_flags & DTE_INVALID) + + if (IS_CHECKER(ck)) { + /** Check flags since this structure has no magic. */ + if ((dae_df->dae_flags & ~DTE_FLAGS_VALID) != 0 || + DAOS_FAIL_CHECK(DAOS_FAULT_DAE_INV_FLAGS)) { + CK_PRINTF(ck, "[%d] invalid flags (%#" PRIx16 ")\n", i, + dae_df->dae_flags); + D_GOTO(out, rc = -DER_IO); + } + } + + if (dae_df->dae_flags & DTE_INVALID) { + CK_PRINTF(ck, "[%d] flagged invalid.\n", i); continue; + } + CK_PRINTF(ck, "[%d] ID (dae_xid)... ", i); if (daos_is_zero_dti(&dae_df->dae_xid)) { + CK_APPENDL_WARN(ck, "zero"); D_WARN("Hit zero active DTX entry.\n"); continue; } + CK_APPENDL_OK(ck); + CK_PRINTF(ck, "[%d] LID (dae_lid)... ", i); if (dae_df->dae_lid < DTX_LID_RESERVED) { + CK_APPENDFL_ERR(ck, "%" PRIu32 " >= %d", dae_df->dae_lid, + DTX_LID_RESERVED); D_ERROR("Corruption in DTX table found, lid=%d" " is invalid\n", dae_df->dae_lid); D_GOTO(out, rc = -DER_IO); } - rc = lrua_allocx_inplace(cont->vc_dtx_array, - dae_df->dae_lid - DTX_LID_RESERVED, - dae_df->dae_epoch, &dae); + + if (!DAOS_FAIL_CHECK(DAOS_FAULT_DAE_ALLOC)) { + rc = lrua_allocx_inplace(cont->vc_dtx_array, + dae_df->dae_lid - DTX_LID_RESERVED, + dae_df->dae_epoch, &dae); + } else { + rc = daos_errno2der(daos_fail_value_get()); + } if (rc != 0) { if (rc == -DER_NOMEM) { + CK_APPENDL_RC(ck, rc); D_ERROR("Not enough memory for DTX " "table\n"); } else { + CK_APPENDFL_ERR(ck, + "cannot append to the active DTX table " + "(dae_lid=%" PRIu32 ", dae_epoch=%#" PRIx64 + ")", + dae_df->dae_lid, dae_df->dae_epoch); D_ERROR("Corruption in DTX table found," " lid=%d is invalid rc="DF_RC "\n", dae_df->dae_lid, @@ -3660,6 +3721,7 @@ vos_dtx_act_reindex(struct vos_container *cont) } D_GOTO(out, rc); } + CK_APPENDL_OK(ck); D_ASSERT(dae != NULL); D_DEBUG(DB_TRACE, "Re-indexed lid DTX: "DF_DTI @@ -3681,30 +3743,38 @@ vos_dtx_act_reindex(struct vos_container *cont) int count; count = DAE_REC_CNT(dae) - DTX_INLINE_REC_CNT; + CK_PRINTF(ck, "[%d] %d non-inlined records... ", i, count); size = sizeof(*dae->dae_records) * count; D_ALLOC_NZ(dae->dae_records, size); if (dae->dae_records == NULL) { + rc = -DER_NOMEM; + CK_APPENDL_RC(ck, rc); dtx_evict_lid(cont, dae); - D_GOTO(out, rc = -DER_NOMEM); + goto out; } memcpy(dae->dae_records, umem_off2ptr(umm, dae_df->dae_rec_off), size); dae->dae_rec_cap = count; + + CK_APPENDL_OK(ck); } + CK_PRINTF(ck, "[%d] Append to active DTX tree... ", i); d_iov_set(&kiov, &DAE_XID(dae), sizeof(DAE_XID(dae))); d_iov_set(&riov, dae, sizeof(*dae)); rc = dbtree_upsert(cont->vc_dtx_active_hdl, BTR_PROBE_EQ, DAOS_INTENT_UPDATE, &kiov, &riov, NULL); if (rc != 0) { + CK_APPENDL_RC(ck, rc); D_FREE(dae->dae_records); dtx_evict_lid(cont, dae); goto out; } + CK_APPENDL_OK(ck); dae->dae_start_time = start_time; d_list_add_tail(&dae->dae_link, &cont->vc_dtx_act_list); @@ -3748,7 +3818,14 @@ vos_dtx_act_reindex(struct vos_container *cont) dbd_count++; } - if (unlikely(dbd_count != dbd->dbd_count)) { + if (IS_CHECKER(ck) && dbd->dbd_index > 0) { + checker_print_indent_dec(ck); + } + + if (unlikely(dbd_count != dbd->dbd_count) || + DAOS_FAIL_CHECK(DAOS_FAULT_DBD_COUNT)) { + CK_PRINTF(ck, CK_DBD_FMT "expected %d active entries, but found %d.\n", + dbd_off, dbd->dbd_count, dbd_count); D_ERROR("Unmatched active DTX count %d/%d, cap %d, idx %d for blob %p (" UMOFF_PF"), head "UMOFF_PF", tail "UMOFF_PF" in pool " DF_UUID" cont "DF_UUID"\n", dbd_count, dbd->dbd_count, dbd->dbd_cap, @@ -3759,12 +3836,22 @@ vos_dtx_act_reindex(struct vos_container *cont) D_GOTO(out, rc = -DER_IO); } + checker_print_indent_dec(ck); + CK_PRINTF(ck, CK_DBD_FMT CHECKER_OK_INFIX ".\n", dbd_off); + dbd_off = dbd->dbd_next; } cont->vc_dtx_reindex_eph_diff = diff; out: + if (IS_CHECKER(ck)) { + /** Restore the checker level in case the control flow jumped here from inside the + * loops above. */ + ck->ck_level = ck_level_cache; + ck->ck_indent_set(ck); + } + return rc > 0 ? 0 : rc; } @@ -4158,7 +4245,7 @@ vos_dtx_cache_reset(daos_handle_t coh, bool force) return rc; } - rc = vos_dtx_act_reindex(cont); + rc = vos_dtx_act_reindex(cont, NULL); if (rc != 0) { D_ERROR("Fail to reindex active DTX table for "DF_UUID": "DF_RC"\n", DP_UUID(cont->vc_id), DP_RC(rc)); diff --git a/src/vos/vos_gc.c b/src/vos/vos_gc.c index 8e1cb8790d0..128952a7f95 100644 --- a/src/vos/vos_gc.c +++ b/src/vos/vos_gc.c @@ -1516,8 +1516,8 @@ gc_open_bkt(struct umem_attr *uma, struct vos_gc_bkt_df *bkt_df, struct checker if (IS_CHECKER(ck)) { CK_PRINT(ck, CK_GC_TREE_STR "...\n"); - CK_INDENT(ck, rc = dbtree_check_inplace(&bkt_df->gd_bins_root, uma, ck_report, ck, - error_on_non_zero_padding)); + CK_INDENT(ck, rc = dbtree_check_inplace(&bkt_df->gd_bins_root, uma, NULL, ck_report, + ck, error_on_non_zero_padding)); CK_PRINTL_RC(ck, rc, CK_GC_TREE_STR); if (rc != DER_SUCCESS) { return rc; diff --git a/src/vos/vos_internal.h b/src/vos/vos_internal.h index 946a3e17ac2..765b1f9be99 100644 --- a/src/vos/vos_internal.h +++ b/src/vos/vos_internal.h @@ -845,11 +845,12 @@ vos_dtx_post_handle(struct vos_container *cont, struct vos_dtx_act_ent **daes, b * Establish indexed active DTX table in DRAM. * * \param cont [IN] Pointer to the container. + * \param ck [IN] Checker structure. * * \return 0 on success and negative on failure. */ int -vos_dtx_act_reindex(struct vos_container *cont); +vos_dtx_act_reindex(struct vos_container *cont, struct checker *ck); int vos_dtx_record_oid(struct dtx_handle *dth, struct vos_container *cont, daos_unit_oid_t oid); diff --git a/src/vos/vos_iterator.c b/src/vos/vos_iterator.c index 71cc4a1e4d9..0a7b19850fb 100644 --- a/src/vos/vos_iterator.c +++ b/src/vos/vos_iterator.c @@ -1169,3 +1169,9 @@ vos_iterate(vos_iter_param_t *param, vos_iter_type_t type, bool recursive, return vos_iterate_internal(param, type, recursive, false, anchors, pre_cb, post_cb, arg, dth); } + +int +vos_iter_check(daos_handle_t ih, vos_iter_entry_t *entry, vos_iter_type_t type, struct checker *ck) +{ + return 0; +} diff --git a/src/vos/vos_obj_index.c b/src/vos/vos_obj_index.c index 328dc77fdf4..d46018f91a2 100644 --- a/src/vos/vos_obj_index.c +++ b/src/vos/vos_obj_index.c @@ -217,16 +217,57 @@ oi_node_alloc(struct btr_instance *tins, int size) return umem_zalloc(&tins->ti_umm, size); } +#define REP_OBJECT_FMT "Object (oid=" DF_UOID ")... " + +static int +oi_rec_check(struct btr_instance *tins, struct btr_record *rec, report_fn_t report_fn, + void *report_arg) +{ + d_iov_t val_iov; + struct vos_obj_df *obj; + int rc; + + report_fn(report_arg, REPORT_MSG, "Record fetch (off=%#lx)... ", rec->rec_off); + rc = tins->ti_ops->to_rec_fetch(tins, rec, NULL, &val_iov); + if (rc != DER_SUCCESS) { + report_fn(report_arg, REPORT_ERROR | REPORT_NO_PREFIX, DF_RC "\n", DP_RC(rc)); + return rc; + } + report_fn(report_arg, REPORT_MSG | REPORT_NO_PREFIX, CHECKER_OK_INFIX ".\n"); + + D_ASSERT(val_iov.iov_buf != NULL); + D_ASSERT(val_iov.iov_len == vos_obj_df_size((struct vos_pool *)tins->ti_priv)); + + obj = val_iov.iov_buf; + + report_fn(report_arg, REPORT_INDENT_INC, NULL); + report_fn(report_arg, REPORT_MSG, REP_OBJECT_FMT "\n", DP_UOID(obj->vo_id)); + report_fn(report_arg, REPORT_INDENT_INC, NULL); + rc = ilog_root_is_valid(&obj->vo_ilog, report_fn, report_arg); + report_fn(report_arg, REPORT_INDENT_DEC, NULL); + if (rc == DER_SUCCESS) { + report_fn(report_arg, REPORT_MSG, REP_OBJECT_FMT CHECKER_OK_INFIX ".\n", + DP_UOID(obj->vo_id)); + } else { + report_fn(report_arg, REPORT_ERROR, REP_OBJECT_FMT DF_RC ".\n", DP_UOID(obj->vo_id), + DP_RC(rc)); + } + report_fn(report_arg, REPORT_INDENT_DEC, NULL); + + return rc; +} + static btr_ops_t oi_btr_ops = { - .to_rec_msize = oi_rec_msize, - .to_hkey_size = oi_hkey_size, - .to_hkey_gen = oi_hkey_gen, - .to_hkey_cmp = oi_hkey_cmp, - .to_rec_alloc = oi_rec_alloc, - .to_rec_free = oi_rec_free, - .to_rec_fetch = oi_rec_fetch, - .to_rec_update = oi_rec_update, - .to_node_alloc = oi_node_alloc, + .to_rec_msize = oi_rec_msize, + .to_hkey_size = oi_hkey_size, + .to_hkey_gen = oi_hkey_gen, + .to_hkey_cmp = oi_hkey_cmp, + .to_rec_alloc = oi_rec_alloc, + .to_rec_free = oi_rec_free, + .to_rec_fetch = oi_rec_fetch, + .to_rec_update = oi_rec_update, + .to_node_alloc = oi_node_alloc, + .to_rec_check = oi_rec_check, }; bool diff --git a/src/vos/vos_pool.c b/src/vos/vos_pool.c index e856a84e004..fea565bf089 100644 --- a/src/vos/vos_pool.c +++ b/src/vos/vos_pool.c @@ -1693,7 +1693,7 @@ pool_open_post(struct umem_pool **p_ph, struct vos_pool_df *pool_df, unsigned in if (IS_CHECKER(ck)) { CK_PRINT(ck, CK_CONT_TREE_STR "...\n"); - CK_INDENT(ck, rc = dbtree_check_inplace(&pool_df->pd_cont_root, &pool->vp_uma, + CK_INDENT(ck, rc = dbtree_check_inplace(&pool_df->pd_cont_root, &pool->vp_uma, pool, ck_report, ck, error_on_non_zero_padding)); CK_PRINTL_RC(ck, rc, CK_CONT_TREE_STR); if (rc != DER_SUCCESS) { @@ -1819,6 +1819,7 @@ vos_pool_open_metrics(const char *path, uuid_t uuid, unsigned int flags, void *m } pool->vp_opened++; *poh = vos_pool2hdl(pool); + checker_print_indent_dec(ck); return 0; }