src/hmem: Added handle field to close_handle

Add a void **handle to close_handle for level-zero to use in its new ipc protocol Signed-off-by: Zach Dworkin <[email protected]>
zachdworkin · Aug 2, 2024 · 99fa453 · 99fa453
1 parent 8f27d27
commit 99fa453
Show file tree

Hide file tree

Showing 7 changed files with 20 additions and 16 deletions.
diff --git a/include/ofi_hmem.h b/include/ofi_hmem.h
@@ -116,7 +116,7 @@ struct ofi_hmem_ops {
 	int (*get_handle)(void *base_addr, size_t base_length, void **handle);
 	int (*open_handle)(void **handle, size_t base_length, uint64_t device,
 			   void **mapped_addr);
-	int (*close_handle)(void *mapped_addr);
+	int (*close_handle)(void *mapped_addr, void **handle);
 	int (*host_register)(void *addr, size_t size);
 	int (*host_unregister)(void *addr);
 	int (*get_base_addr)(const void *addr, size_t len, void **base_addr,
@@ -149,7 +149,7 @@ int rocr_get_base_addr(const void *ptr, size_t len, void **base, size_t *size);
 int rocr_get_handle(void *dev_buf, size_t size, void **handle);
 int rocr_open_handle(void **handle, size_t size, uint64_t device,
 		     void **ipc_ptr);
-int rocr_close_handle(void *ipc_ptr);
+int rocr_close_handle(void *ipc_ptr, void **handle);
 bool rocr_is_ipc_enabled(void);
 int rocr_create_async_copy_event(uint64_t device,
 				 ofi_hmem_async_event_t *event);
@@ -184,7 +184,7 @@ int cuda_dev_reg_copy_from_hmem(uint64_t handle, void *dest, const void *src,
 int cuda_get_handle(void *dev_buf, size_t size, void **handle);
 int cuda_open_handle(void **handle, size_t size, uint64_t device,
 		     void **ipc_ptr);
-int cuda_close_handle(void *ipc_ptr);
+int cuda_close_handle(void *ipc_ptr, void **handle);
 int cuda_get_base_addr(const void *ptr, size_t len, void **base, size_t *size);
 
 bool cuda_is_ipc_enabled(void);
@@ -215,7 +215,7 @@ int ze_hmem_get_shared_handle(uint64_t device, void *dev_buf, int *ze_fd,
 			      void **handle);
 int ze_hmem_open_shared_handle(uint64_t device, int *peer_fds, void **handle,
 			       int *ze_fd, void **ipc_ptr);
-int ze_hmem_close_handle(void *ipc_ptr);
+int ze_hmem_close_handle(void *ipc_ptr, void **handle);
 bool ze_hmem_p2p_enabled(void);
 int ze_hmem_get_ipc_handle_size(size_t *size);
 int ze_hmem_get_base_addr(const void *ptr, size_t len, void **base,
@@ -309,7 +309,7 @@ static inline int ofi_hmem_no_open_handle(void **handle, size_t size,
 	return -FI_ENOSYS;
 }
 
-static inline int ofi_hmem_no_close_handle(void *mapped_addr)
+static inline int ofi_hmem_no_close_handle(void *mapped_addr, void **handle)
 {
 	return -FI_ENOSYS;
 }
@@ -416,7 +416,8 @@ int ofi_hmem_get_handle(enum fi_hmem_iface iface, void *base_addr,
 			size_t size, void **handle);
 int ofi_hmem_open_handle(enum fi_hmem_iface iface, void **handle,
 			 size_t size, uint64_t device, void **mapped_addr);
-int ofi_hmem_close_handle(enum fi_hmem_iface iface, void *mapped_addr);
+int ofi_hmem_close_handle(enum fi_hmem_iface iface, void *mapped_addr,
+			  void **handle);
 int ofi_hmem_get_base_addr(enum fi_hmem_iface iface, const void *addr,
 			   size_t len, void **base_addr, size_t *base_length);
 bool ofi_hmem_is_initialized(enum fi_hmem_iface iface);

diff --git a/prov/shm/src/smr_progress.c b/prov/shm/src/smr_progress.c
@@ -597,7 +597,8 @@ static struct smr_pend_entry *smr_progress_ipc(struct smr_cmd *cmd,
 	if (cmd->msg.data.ipc_info.iface == FI_HMEM_ZE) {
 		close(ipc_fd);
 		/* Truncation error takes precedence over close_handle error */
-		ret = ofi_hmem_close_handle(cmd->msg.data.ipc_info.iface, base);
+		ret = ofi_hmem_close_handle(cmd->msg.data.ipc_info.iface, base,
+					    NULL);
 	} else {
 		ofi_mr_cache_delete(domain->ipc_cache, mr_entry);
 	}

diff --git a/src/hmem.c b/src/hmem.c
@@ -568,9 +568,10 @@ int ofi_hmem_open_handle(enum fi_hmem_iface iface, void **handle,
 					   mapped_addr);
 }
 
-int ofi_hmem_close_handle(enum fi_hmem_iface iface, void *mapped_addr)
+int ofi_hmem_close_handle(enum fi_hmem_iface iface, void *mapped_addr,
+			  void **handle)
 {
-	return hmem_ops[iface].close_handle(mapped_addr);
+	return hmem_ops[iface].close_handle(mapped_addr, handle);
 }
 
 int ofi_hmem_get_base_addr(enum fi_hmem_iface iface, const void *addr,

diff --git a/src/hmem_cuda.c b/src/hmem_cuda.c
@@ -420,7 +420,7 @@ int cuda_open_handle(void **handle, size_t size, uint64_t device,
 	return (cuda_ret == cudaErrorAlreadyMapped) ? -FI_EALREADY:-FI_EINVAL;
 }
 
-int cuda_close_handle(void *ipc_ptr)
+int cuda_close_handle(void *ipc_ptr, void **handle)
 {
 	cudaError_t cuda_ret;
 
@@ -1009,7 +1009,7 @@ int cuda_open_handle(void **handle, size_t size, uint64_t device,
 	return -FI_ENOSYS;
 }
 
-int cuda_close_handle(void *ipc_ptr)
+int cuda_close_handle(void *ipc_ptr, void **handle)
 {
 	return -FI_ENOSYS;
 }

diff --git a/src/hmem_ipc_cache.c b/src/hmem_ipc_cache.c
@@ -75,7 +75,8 @@ static void ipc_cache_delete_region(struct ofi_mr_cache *cache,
 				     struct ofi_mr_entry *entry)
 {
 	ofi_hmem_close_handle(entry->info.iface,
-			      entry->info.mapped_addr);
+			      entry->info.mapped_addr,
+			      (void **) &entry->info.handle);
 }
 
 /**

diff --git a/src/hmem_rocr.c b/src/hmem_rocr.c
@@ -672,7 +672,7 @@ int rocr_open_handle(void **handle, size_t len, uint64_t device, void **ipc_ptr)
 	return -FI_EINVAL;
 }
 
-int rocr_close_handle(void *ipc_ptr)
+int rocr_close_handle(void *ipc_ptr, void **handle)
 {
 	hsa_status_t hsa_ret;
 
@@ -1229,7 +1229,7 @@ int rocr_open_handle(void **handle, size_t len, uint64_t device, void **ipc_ptr)
 	return -FI_ENOSYS;
 }
 
-int rocr_close_handle(void *ipc_ptr)
+int rocr_close_handle(void *ipc_ptr, void **handle)
 {
 	return -FI_ENOSYS;
 }

diff --git a/src/hmem_ze.c b/src/hmem_ze.c
@@ -1087,7 +1087,7 @@ int ze_hmem_open_handle(void **handle, size_t size, uint64_t device,
 	return FI_SUCCESS;
 }
 
-int ze_hmem_close_handle(void *ipc_ptr)
+int ze_hmem_close_handle(void *ipc_ptr, void **handle)
 {
 	ze_result_t ze_ret;
 
@@ -1413,7 +1413,7 @@ int ze_hmem_open_shared_handle(uint64_t device, int *peer_fds, void **handle,
 	return -FI_ENOSYS;
 }
 
-int ze_hmem_close_handle(void *ipc_ptr)
+int ze_hmem_close_handle(void *ipc_ptr, void **handle)
 {
 	return -FI_ENOSYS;
 }