From 0aba4022ad369dded2c1e3fdaedb5ee300f3dfc2 Mon Sep 17 00:00:00 2001 From: Alexey Marchuk Date: Mon, 13 Mar 2023 13:37:12 +0100 Subject: [PATCH] nvmf/vfio_user: Post SQ delete cpl when qpair is destroyed This patch removes usage of cb_fn argument of spdk_nvmf_qpair_disconnect API. Instead of relying on the callback, post a completion on delete SQ command when transport qpair_fini is called. Signed-off-by: Alexey Marchuk Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/17168 (master) (cherry picked from commit bcd0ea8c1cab97964d437ecb611c61d412546836) Change-Id: I68dec97ea94e89f48a8667da82f88b5e24fc0d88 Signed-off-by: Krzysztof Karas Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/17707 Reviewed-by: Jim Harris Reviewed-by: Konrad Sztyber Reviewed-by: Ben Walker Tested-by: SPDK CI Jenkins --- lib/nvmf/vfio_user.c | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/lib/nvmf/vfio_user.c b/lib/nvmf/vfio_user.c index c6c2b0d40a7..9b1c97a11d8 100644 --- a/lib/nvmf/vfio_user.c +++ b/lib/nvmf/vfio_user.c @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright (C) 2020 Intel Corporation. * Copyright (c) 2019-2022, Nutanix Inc. All rights reserved. - * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ /* @@ -318,6 +318,8 @@ struct nvmf_vfio_user_sq { */ struct spdk_nvme_cmd create_io_sq_cmd; + struct vfio_user_delete_sq_ctx *delete_ctx; + /* Currently unallocated reqs. */ TAILQ_HEAD(, nvmf_vfio_user_req) free_reqs; /* Poll group entry */ @@ -2224,11 +2226,11 @@ handle_create_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, } /* For ADMIN I/O DELETE SUBMISSION QUEUE the NVMf library will disconnect and free - * queue pair, so save the command in a context. + * queue pair, so save the command id and controller in a context. */ struct vfio_user_delete_sq_ctx { struct nvmf_vfio_user_ctrlr *vu_ctrlr; - struct spdk_nvme_cmd delete_io_sq_cmd; + uint16_t cid; }; static void @@ -2247,7 +2249,7 @@ vfio_user_qpair_delete_cb(void *cb_arg) cb_arg); } else { post_completion(vu_ctrlr, admin_cq, 0, 0, - ctx->delete_io_sq_cmd.cid, + ctx->cid, SPDK_NVME_SC_SUCCESS, SPDK_NVME_SCT_GENERIC); free(ctx); } @@ -2264,7 +2266,6 @@ handle_del_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, uint16_t sc = SPDK_NVME_SC_SUCCESS; struct nvmf_vfio_user_sq *sq; struct nvmf_vfio_user_cq *cq; - struct vfio_user_delete_sq_ctx *ctx; SPDK_DEBUGLOG(nvmf_vfio, "%s: delete I/O %cqid:%d\n", ctrlr_id(ctrlr), is_cq ? 'c' : 's', @@ -2293,21 +2294,20 @@ handle_del_io_q(struct nvmf_vfio_user_ctrlr *ctrlr, * VM reboot or CC.EN change, so we have to delete it in all * other cases. */ - ctx = calloc(1, sizeof(*ctx)); - if (!ctx) { + sq = ctrlr->sqs[cmd->cdw10_bits.delete_io_q.qid]; + sq->delete_ctx = calloc(1, sizeof(*sq->delete_ctx)); + if (!sq->delete_ctx) { sct = SPDK_NVME_SCT_GENERIC; sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR; goto out; } - ctx->vu_ctrlr = ctrlr; - ctx->delete_io_sq_cmd = *cmd; - - sq = ctrlr->sqs[cmd->cdw10_bits.delete_io_q.qid]; + sq->delete_ctx->vu_ctrlr = ctrlr; + sq->delete_ctx->cid = cmd->cid; sq->sq_state = VFIO_USER_SQ_DELETED; assert(ctrlr->cqs[sq->cqid]->cq_ref); ctrlr->cqs[sq->cqid]->cq_ref--; - spdk_nvmf_qpair_disconnect(&sq->qpair, vfio_user_qpair_delete_cb, ctx); + spdk_nvmf_qpair_disconnect(&sq->qpair, NULL, NULL); return 0; } @@ -5271,11 +5271,14 @@ nvmf_vfio_user_close_qpair(struct spdk_nvmf_qpair *qpair, struct nvmf_vfio_user_sq *sq; struct nvmf_vfio_user_ctrlr *vu_ctrlr; struct nvmf_vfio_user_endpoint *endpoint; + struct vfio_user_delete_sq_ctx *del_ctx; assert(qpair != NULL); sq = SPDK_CONTAINEROF(qpair, struct nvmf_vfio_user_sq, qpair); vu_ctrlr = sq->ctrlr; endpoint = vu_ctrlr->endpoint; + del_ctx = sq->delete_ctx; + sq->delete_ctx = NULL; pthread_mutex_lock(&endpoint->lock); TAILQ_REMOVE(&vu_ctrlr->connected_sqs, sq, tailq); @@ -5294,6 +5297,10 @@ nvmf_vfio_user_close_qpair(struct spdk_nvmf_qpair *qpair, } pthread_mutex_unlock(&endpoint->lock); + if (del_ctx) { + vfio_user_qpair_delete_cb(del_ctx); + } + if (cb_fn) { cb_fn(cb_arg); }