From 668958f3c1617f18e04ffee099656e7fb2effa94 Mon Sep 17 00:00:00 2001 From: Stephen Hemminger Date: Tue, 6 Sep 2022 09:45:22 -0700 Subject: [PATCH] eal: fix data race in multi-process support If DPDK is built with thread sanitizer it reports a race in setting of multiprocess file descriptor. The fix is to use atomic operations when updating mp_fd. Build: $ meson -Db_sanitize=address build $ ninja -C build Simple example: $ .build/app/dpdk-testpmd -l 1-3 --no-huge EAL: Detected CPU lcores: 16 EAL: Detected NUMA nodes: 1 EAL: Static memory layout is selected, amount of reserved memory can be adjusted with -m or --socket-mem EAL: Detected static linkage of DPDK EAL: Multi-process socket /run/user/1000/dpdk/rte/mp_socket EAL: Selected IOVA mode 'VA' testpmd: No probed ethernet devices testpmd: create a new mbuf pool : n=163456, size=2176, socket=0 testpmd: preferred mempool ops selected: ring_mp_mc EAL: Error - exiting with code: 1 Cause: Creation of mbuf pool for socket 0 failed: Cannot allocate memory ================== WARNING: ThreadSanitizer: data race (pid=87245) Write of size 4 at 0x558e04d8ff70 by main thread: #0 rte_mp_channel_cleanup (dpdk-testpmd+0x1e7d30c) #1 rte_eal_cleanup (dpdk-testpmd+0x1e85929) #2 rte_exit (dpdk-testpmd+0x1e5bc0a) #3 mbuf_pool_create.cold (dpdk-testpmd+0x274011) #4 main (dpdk-testpmd+0x5cc15d) Previous read of size 4 at 0x558e04d8ff70 by thread T2: #0 mp_handle (dpdk-testpmd+0x1e7c439) #1 ctrl_thread_init (dpdk-testpmd+0x1e6ee1e) As if synchronized via sleep: #0 nanosleep libsanitizer/tsan/tsan_interceptors_posix.cpp:366 #1 get_tsc_freq (dpdk-testpmd+0x1e92ff9) #2 set_tsc_freq (dpdk-testpmd+0x1e6f2fc) #3 rte_eal_timer_init (dpdk-testpmd+0x1e931a4) #4 rte_eal_init.cold (dpdk-testpmd+0x29e578) #5 main (dpdk-testpmd+0x5cbc45) Location is global 'mp_fd' of size 4 at 0x558e04d8ff70 (dpdk-testpmd+0x000003122f70) Thread T2 'rte_mp_handle' (tid=87248, running) created by main thread at: #0 pthread_create libsanitizer/tsan/tsan_interceptors_posix.cpp:969 #1 rte_ctrl_thread_create (dpdk-testpmd+0x1e6efd0) #2 rte_mp_channel_init.cold (dpdk-testpmd+0x29cb7c) #3 rte_eal_init (dpdk-testpmd+0x1e8662e) #4 main (dpdk-testpmd+0x5cbc45) SUMMARY: ThreadSanitizer: data race (app/dpdk-testpmd+0x1e7d30c) in rte_mp_channel_cleanup ================== ThreadSanitizer: reported 1 warnings Fixes: bacaa2754017 ("eal: add channel for multi-process communication") Cc: stable@dpdk.org Signed-off-by: Stephen Hemminger Acked-by: Anatoly Burakov Reviewed-by: Chengwen Feng --- lib/eal/common/eal_common_proc.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/eal/common/eal_common_proc.c b/lib/eal/common/eal_common_proc.c index 313060528f..1fc1d6c53b 100644 --- a/lib/eal/common/eal_common_proc.c +++ b/lib/eal/common/eal_common_proc.c @@ -260,7 +260,7 @@ rte_mp_action_unregister(const char *name) } static int -read_msg(struct mp_msg_internal *m, struct sockaddr_un *s) +read_msg(int fd, struct mp_msg_internal *m, struct sockaddr_un *s) { int msglen; struct iovec iov; @@ -281,7 +281,7 @@ read_msg(struct mp_msg_internal *m, struct sockaddr_un *s) msgh.msg_controllen = sizeof(control); retry: - msglen = recvmsg(mp_fd, &msgh, 0); + msglen = recvmsg(fd, &msgh, 0); /* zero length message means socket was closed */ if (msglen == 0) @@ -390,11 +390,12 @@ mp_handle(void *arg __rte_unused) { struct mp_msg_internal msg; struct sockaddr_un sa; + int fd; - while (mp_fd >= 0) { + while ((fd = __atomic_load_n(&mp_fd, __ATOMIC_RELAXED)) >= 0) { int ret; - ret = read_msg(&msg, &sa); + ret = read_msg(fd, &msg, &sa); if (ret <= 0) break; @@ -638,9 +639,8 @@ rte_mp_channel_init(void) NULL, mp_handle, NULL) < 0) { RTE_LOG(ERR, EAL, "failed to create mp thread: %s\n", strerror(errno)); - close(mp_fd); close(dir_fd); - mp_fd = -1; + close(__atomic_exchange_n(&mp_fd, -1, __ATOMIC_RELAXED)); return -1; } @@ -656,11 +656,10 @@ rte_mp_channel_cleanup(void) { int fd; - if (mp_fd < 0) + fd = __atomic_exchange_n(&mp_fd, -1, __ATOMIC_RELAXED); + if (fd < 0) return; - fd = mp_fd; - mp_fd = -1; pthread_cancel(mp_handle_tid); pthread_join(mp_handle_tid, NULL); close_socket_fd(fd);