nvmf/vfio_user: Post SQ delete cpl when qpair is destroyed

This patch removes usage of cb_fn argument of
spdk_nvmf_qpair_disconnect API. Instead of relying
on the callback, post a completion on delete SQ
command when transport qpair_fini is called.

Signed-off-by: Alexey Marchuk <[email protected]>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/17168 (master)

(cherry picked from commit bcd0ea8)
Change-Id: I68dec97ea94e89f48a8667da82f88b5e24fc0d88
Signed-off-by: Krzysztof Karas <[email protected]>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/17707
Reviewed-by: Jim Harris <[email protected]>
Reviewed-by: Konrad Sztyber <[email protected]>
Reviewed-by: Ben Walker <[email protected]>
Tested-by: SPDK CI Jenkins <[email protected]>

lib/nvmf/vfio_user.c

@@ -1,7 +1,7 @@
 /*   SPDX-License-Identifier: BSD-3-Clause
  *   Copyright (C) 2020 Intel Corporation.
  *   Copyright (c) 2019-2022, Nutanix Inc. All rights reserved.
- *   Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ *   Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  */
 
 /*
@@ -318,6 +318,8 @@ struct nvmf_vfio_user_sq {
 	 */
 	struct spdk_nvme_cmd			create_io_sq_cmd;
 
+	struct vfio_user_delete_sq_ctx		*delete_ctx;
+
 	/* Currently unallocated reqs. */
 	TAILQ_HEAD(, nvmf_vfio_user_req)	free_reqs;
 	/* Poll group entry */
@@ -2224,11 +2226,11 @@ handle_create_io_q(struct nvmf_vfio_user_ctrlr *ctrlr,
 }
 
 /* For ADMIN I/O DELETE SUBMISSION QUEUE the NVMf library will disconnect and free
- * queue pair, so save the command in a context.
+ * queue pair, so save the command id and controller in a context.
  */
 struct vfio_user_delete_sq_ctx {
 	struct nvmf_vfio_user_ctrlr *vu_ctrlr;
-	struct spdk_nvme_cmd delete_io_sq_cmd;
+	uint16_t cid;
 };
 
 static void
@@ -2247,7 +2249,7 @@ vfio_user_qpair_delete_cb(void *cb_arg)
 				     cb_arg);
 	} else {
 		post_completion(vu_ctrlr, admin_cq, 0, 0,
-				ctx->delete_io_sq_cmd.cid,
+				ctx->cid,
 				SPDK_NVME_SC_SUCCESS, SPDK_NVME_SCT_GENERIC);
 		free(ctx);
 	}
@@ -2264,7 +2266,6 @@ handle_del_io_q(struct nvmf_vfio_user_ctrlr *ctrlr,
 	uint16_t sc = SPDK_NVME_SC_SUCCESS;
 	struct nvmf_vfio_user_sq *sq;
 	struct nvmf_vfio_user_cq *cq;
-	struct vfio_user_delete_sq_ctx *ctx;
 
 	SPDK_DEBUGLOG(nvmf_vfio, "%s: delete I/O %cqid:%d\n",
 		      ctrlr_id(ctrlr), is_cq ? 'c' : 's',
@@ -2293,21 +2294,20 @@ handle_del_io_q(struct nvmf_vfio_user_ctrlr *ctrlr,
 		 * VM reboot or CC.EN change, so we have to delete it in all
 		 * other cases.
 		 */
-		ctx = calloc(1, sizeof(*ctx));
-		if (!ctx) {
+		sq = ctrlr->sqs[cmd->cdw10_bits.delete_io_q.qid];
+		sq->delete_ctx = calloc(1, sizeof(*sq->delete_ctx));
+		if (!sq->delete_ctx) {
 			sct = SPDK_NVME_SCT_GENERIC;
 			sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
 			goto out;
 		}
-		ctx->vu_ctrlr = ctrlr;
-		ctx->delete_io_sq_cmd = *cmd;
-
-		sq = ctrlr->sqs[cmd->cdw10_bits.delete_io_q.qid];
+		sq->delete_ctx->vu_ctrlr = ctrlr;
+		sq->delete_ctx->cid = cmd->cid;
 		sq->sq_state = VFIO_USER_SQ_DELETED;
 		assert(ctrlr->cqs[sq->cqid]->cq_ref);
 		ctrlr->cqs[sq->cqid]->cq_ref--;
 
-		spdk_nvmf_qpair_disconnect(&sq->qpair, vfio_user_qpair_delete_cb, ctx);
+		spdk_nvmf_qpair_disconnect(&sq->qpair, NULL, NULL);
 		return 0;
 	}
 
@@ -5271,11 +5271,14 @@ nvmf_vfio_user_close_qpair(struct spdk_nvmf_qpair *qpair,
 	struct nvmf_vfio_user_sq *sq;
 	struct nvmf_vfio_user_ctrlr *vu_ctrlr;
 	struct nvmf_vfio_user_endpoint *endpoint;
+	struct vfio_user_delete_sq_ctx *del_ctx;
 
 	assert(qpair != NULL);
 	sq = SPDK_CONTAINEROF(qpair, struct nvmf_vfio_user_sq, qpair);
 	vu_ctrlr = sq->ctrlr;
 	endpoint = vu_ctrlr->endpoint;
+	del_ctx = sq->delete_ctx;
+	sq->delete_ctx = NULL;
 
 	pthread_mutex_lock(&endpoint->lock);
 	TAILQ_REMOVE(&vu_ctrlr->connected_sqs, sq, tailq);
@@ -5294,6 +5297,10 @@ nvmf_vfio_user_close_qpair(struct spdk_nvmf_qpair *qpair,
 	}
 	pthread_mutex_unlock(&endpoint->lock);
 
+	if (del_ctx) {
+		vfio_user_qpair_delete_cb(del_ctx);
+	}
+
 	if (cb_fn) {
 		cb_fn(cb_arg);
 	}