Skip to content

rdma/mlx5_dv: stamp RoCE IPv4 TOS from nvme/nvmf transport opts #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jleeh merged 1 commit into from
Aug 28, 2026
Merged

rdma/mlx5_dv: stamp RoCE IPv4 TOS from nvme/nvmf transport opts #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions include/spdk_internal/rdma_provider.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ struct spdk_rdma_provider_qp_init_attr {
struct ibv_pd *pd;
struct spdk_rdma_provider_qp_stats *stats;
spdk_memory_domain_transfer_data_cb domain_transfer;
/* IPv4 TOS / IPv6 TCLASS. 0 = untagged (rdma-core default). */
uint8_t tos;
};

struct spdk_rdma_provider_send_wr_list {
Expand All @@ -56,6 +58,7 @@ struct spdk_rdma_provider_qp {
struct spdk_rdma_provider_recv_wr_list recv_wrs;
struct spdk_rdma_provider_qp_stats *stats;
bool shared_stats;
uint8_t tos;
};

struct spdk_rdma_provider_srq_init_attr {
Expand Down
1 change: 1 addition & 0 deletions lib/nvme/nvme_rdma.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -819,6 +819,7 @@ nvme_rdma_qpair_init(struct nvme_rdma_qpair *rqpair)
attr.cap.max_recv_sge = spdk_min(NVME_RDMA_DEFAULT_RX_SGE, dev_attr.max_sge);
attr.domain_transfer = spdk_rdma_provider_accel_sequence_supported() ?
nvme_rdma_memory_domain_transfer_data : NULL;
attr.tos = rqpair->qpair.ctrlr->opts.transport_tos;

rqpair->rdma_qp = spdk_rdma_provider_qp_create(rqpair->cm_id, &attr);

Expand Down
2 changes: 2 additions & 0 deletions lib/nvmf/rdma.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1042,6 +1042,7 @@ nvmf_rdma_qpair_initialize(struct spdk_nvmf_qpair *qpair)

rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair);
device = rqpair->device;
rtransport = SPDK_CONTAINEROF(qpair->transport, struct spdk_nvmf_rdma_transport, transport);

qp_init_attr.qp_context = rqpair;
qp_init_attr.pd = device->pd;
Expand All @@ -1059,6 +1060,7 @@ nvmf_rdma_qpair_initialize(struct spdk_nvmf_qpair *qpair)
qp_init_attr.cap.max_send_sge = spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_TX_SGE);
qp_init_attr.cap.max_recv_sge = spdk_min((uint32_t)device->attr.max_sge, NVMF_DEFAULT_RX_SGE);
qp_init_attr.stats = &rqpair->poller->stat.qp_stats;
qp_init_attr.tos = rtransport->rdma_opts.tos;

if (rqpair->srq == NULL && nvmf_rdma_resize_cq(rqpair, device) < 0) {
SPDK_ERRLOG("Failed to resize the completion queue. Cannot initialize qpair.\n");
Expand Down
31 changes: 15 additions & 16 deletions lib/rdma_provider/rdma_provider_mlx5_dv.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,30 +26,28 @@ struct spdk_rdma_mlx5_dv_qp {
/*
* rdma_set_option(RDMA_OPTION_ID_TOS) programs SL (SQ / tx_prio). mlx5_dv
* moves the QP itself via ibv_modify_qp, so ConnectX builds the RoCEv2
* IPv4 header from the RTR AV traffic_class. Inferring TOS from ah_attr.sl
* is a no-op: the RTR AV often has sl=0 even when the SQ is prio 3.
*
* Force IEEE 802.1p class 3 → TOS 96 / DSCP 24 (CS3), matching Longhorn
* dataEngineRdmaPriorityClass v2=3. Do not add IBV_QP_AV if the kernel
* did not already supply a full AV.
* IPv4 header from the RTR AV traffic_class. Copy the caller-configured
* TOS (nvme transport_tos / nvmf tos) into that AV. Do not add IBV_QP_AV
* if the kernel did not already supply a full AV. TOS 0 leaves the AV
* untagged.
*/
#define RDMA_MLX5_DV_ROCE_TOS 96
#define RDMA_MLX5_DV_ROCE_SL 3

static void
rdma_mlx5_dv_fill_roce_tos(struct ibv_qp_attr *qp_attr, int qp_attr_mask)
rdma_mlx5_dv_fill_roce_tos(struct ibv_qp_attr *qp_attr, int qp_attr_mask, uint8_t tos)
{
if (tos == 0) {
return;
}
if ((qp_attr_mask & IBV_QP_AV) == 0) {
SPDK_NOTICELOG("RoCE RTR missing IBV_QP_AV, cannot set TOS (mask=0x%x)\n",
qp_attr_mask);
SPDK_NOTICELOG("RoCE RTR missing IBV_QP_AV, cannot set TOS %u (mask=0x%x)\n",
tos, qp_attr_mask);
return;
}

qp_attr->ah_attr.sl = RDMA_MLX5_DV_ROCE_SL;
qp_attr->ah_attr.grh.traffic_class = RDMA_MLX5_DV_ROCE_TOS;
qp_attr->ah_attr.sl = tos >> 5;
qp_attr->ah_attr.grh.traffic_class = tos;

SPDK_NOTICELOG("RoCE RTR TOS %u sl %u tclass %u is_global %d mask=0x%x\n",
RDMA_MLX5_DV_ROCE_TOS,
tos,
qp_attr->ah_attr.sl,
qp_attr->ah_attr.grh.traffic_class,
qp_attr->ah_attr.is_global,
Expand Down Expand Up @@ -82,7 +80,7 @@ rdma_mlx5_dv_init_qpair(struct spdk_rdma_mlx5_dv_qp *mlx5_qp)
return rc;
}

rdma_mlx5_dv_fill_roce_tos(&qp_attr, qp_attr_mask);
rdma_mlx5_dv_fill_roce_tos(&qp_attr, qp_attr_mask, mlx5_qp->common.tos);

rc = ibv_modify_qp(mlx5_qp->common.qp, &qp_attr, qp_attr_mask);
if (rc) {
Expand Down Expand Up @@ -160,6 +158,7 @@ spdk_rdma_provider_qp_create(struct rdma_cm_id *cm_id,

mlx5_qp->common.qp = qp;
mlx5_qp->common.cm_id = cm_id;
mlx5_qp->common.tos = qp_attr->tos;
mlx5_qp->qpex = ibv_qp_to_qp_ex(qp);

if (!mlx5_qp->qpex) {
Expand Down
1 change: 1 addition & 0 deletions lib/rdma_provider/rdma_provider_verbs.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ spdk_rdma_provider_qp_create(struct rdma_cm_id *cm_id,
}
spdk_rdma_qp->qp = cm_id->qp;
spdk_rdma_qp->cm_id = cm_id;
spdk_rdma_qp->tos = qp_attr->tos;
spdk_rdma_qp->domain = spdk_rdma_utils_get_memory_domain(qp_attr->pd);
if (!spdk_rdma_qp->domain) {
spdk_rdma_provider_qp_destroy(spdk_rdma_qp);
Expand Down
Loading