diff --git a/src/core/event/event_handler_manager.cpp b/src/core/event/event_handler_manager.cpp index 2a6f4584c..fcf0cc62d 100644 --- a/src/core/event/event_handler_manager.cpp +++ b/src/core/event/event_handler_manager.cpp @@ -264,8 +264,8 @@ event_handler_manager::event_handler_manager(bool internal_thread_mode) m_epfd = -1; m_event_handler_tid = 0; + m_b_continue_running = true; if (!internal_thread_mode) { - m_b_continue_running = true; return; } @@ -273,29 +273,22 @@ event_handler_manager::event_handler_manager(bool internal_thread_mode) BULLSEYE_EXCLUDE_BLOCK_START if (m_epfd == -1) { evh_logdbg("epoll_create failed on ibv device collection (errno=%d %m)", errno); - free_evh_resources(); throw_xlio_exception("epoll_create failed on ibv device collection"); } BULLSEYE_EXCLUDE_BLOCK_END - m_b_continue_running = true; - wakeup_set_epoll_fd(m_epfd); going_to_sleep(); } event_handler_manager::~event_handler_manager() { - free_evh_resources(); -} - -void event_handler_manager::free_evh_resources() -{ - evh_logfunc(""); - - // Flag thread to stop on next loop stop_thread(); - evh_logfunc("Thread stopped"); + + if (m_epfd >= 0) { + SYSCALL(close, m_epfd); + } + m_epfd = -1; } // event handler main thread startup @@ -344,56 +337,53 @@ int event_handler_manager::start_thread() { cpu_set_t cpu_set; pthread_attr_t tattr; + int ret; + bool affinity_requested = false; if (!m_b_continue_running) { + errno = ECANCELED; return -1; } - if (m_event_handler_tid != 0) { return 0; } - // m_reg_action_q.reserve(); //todo change to vector and reserve - - BULLSEYE_EXCLUDE_BLOCK_START - if (pthread_attr_init(&tattr)) { - evh_logpanic("Failed to initialize thread attributes"); + ret = pthread_attr_init(&tattr); + if (ret != 0) { + return -1; } - BULLSEYE_EXCLUDE_BLOCK_END cpu_set = safe_mce_sys().internal_thread_affinity; if (strcmp(safe_mce_sys().internal_thread_affinity_str, "-1") && - !strcmp(safe_mce_sys().internal_thread_cpuset, - MCE_DEFAULT_INTERNAL_THREAD_CPUSET)) { // no affinity - BULLSEYE_EXCLUDE_BLOCK_START - if (pthread_attr_setaffinity_np(&tattr, sizeof(cpu_set), &cpu_set)) { - evh_logpanic("Failed to set CPU affinity"); + !strcmp(safe_mce_sys().internal_thread_cpuset, MCE_DEFAULT_INTERNAL_THREAD_CPUSET)) { + ret = pthread_attr_setaffinity_np(&tattr, sizeof(cpu_set), &cpu_set); + if (ret != 0) { + evh_logwarn("Failed to set event handler thread affinity. (errno=%d)", errno); } - BULLSEYE_EXCLUDE_BLOCK_END + affinity_requested = (ret == 0); } else { evh_logdbg("Internal thread affinity not set."); } - int ret = pthread_create(&m_event_handler_tid, &tattr, event_handler_thread, this); - if (ret) { - // maybe it's the cset issue? try without affinity - evh_logwarn("Failed to start event handler thread with thread affinity - trying without. " - "[errno=%d %s]", - ret, strerror(ret)); - BULLSEYE_EXCLUDE_BLOCK_START - if (pthread_attr_init(&tattr)) { - evh_logpanic("Failed to initialize thread attributes"); - } - if (pthread_create(&m_event_handler_tid, &tattr, event_handler_thread, this)) { - evh_logpanic("Failed to start event handler thread"); - } - BULLSEYE_EXCLUDE_BLOCK_END + ret = pthread_create(&m_event_handler_tid, &tattr, event_handler_thread, this); + if (ret != 0 && affinity_requested) { + // Try without affinity in case this is a cset issue. + evh_logwarn("Failed to start event handler thread with a thread affinity. Trying default " + "thread affinity. (errno=%d)", + errno); + pthread_attr_destroy(&tattr); + ret = pthread_attr_init(&tattr) + ?: pthread_create(&m_event_handler_tid, &tattr, event_handler_thread, this); } - + // Destroy will either succeed or return EINVAL if the init fails in the above block. pthread_attr_destroy(&tattr); - evh_logdbg("Started event handler thread"); - return 0; + if (ret == 0) { + evh_logdbg("Started event handler thread."); + } else { + evh_logerr("Failed to start event handler thread. (errno=%d)", errno); + } + return ret; } void event_handler_manager::stop_thread() @@ -410,18 +400,12 @@ void event_handler_manager::stop_thread() // Wait for thread exit if (m_event_handler_tid) { pthread_join(m_event_handler_tid, nullptr); - evh_logdbg("event handler thread stopped"); + evh_logdbg("Event handler thread stopped."); } else { - evh_logdbg("event handler thread not running"); + evh_logdbg("Event handler thread not running."); } } m_event_handler_tid = 0; - - // Close main epfd and signaling socket - if (m_epfd >= 0) { - SYSCALL(close, m_epfd); - } - m_epfd = -1; } void event_handler_manager::update_epfd(int fd, int operation, int events) @@ -483,15 +467,11 @@ void event_handler_manager::post_new_reg_action(reg_action_t ®_action) return; } - start_thread(); - evh_logfunc("add event action %s (%d)", reg_action_str(reg_action.type), reg_action.type); - bool is_empty = false; + bool is_empty; m_reg_action_q_lock.lock(); - if (m_p_reg_action_q_to_push_to->empty()) { - is_empty = true; - } + is_empty = m_p_reg_action_q_to_push_to->empty(); m_p_reg_action_q_to_push_to->push_back(reg_action); m_reg_action_q_lock.unlock(); if (is_empty) { diff --git a/src/core/event/event_handler_manager.h b/src/core/event/event_handler_manager.h index 666c7e34e..713b599eb 100644 --- a/src/core/event/event_handler_manager.h +++ b/src/core/event/event_handler_manager.h @@ -208,6 +208,7 @@ class event_handler_manager : public wakeup_pipe { void register_command_event(int fd, command *cmd); void *thread_loop(); + int start_thread(); void stop_thread(); bool is_running() { return m_b_continue_running; }; @@ -251,12 +252,9 @@ class event_handler_manager : public wakeup_pipe { void handle_registration_action(reg_action_t ®_action); void process_ibverbs_event(event_handler_map_t::iterator &i); void process_rdma_cm_event(event_handler_map_t::iterator &i); - int start_thread(); void event_channel_post_process_for_rdma_events(void *p_event); void *event_channel_pre_process_for_rdma_events(void *p_event_channel_handle, void **p_event); - - void free_evh_resources(void); }; extern event_handler_manager *g_p_event_handler_manager; diff --git a/src/core/main.cpp b/src/core/main.cpp index 8b41c4314..8b33557ef 100644 --- a/src/core/main.cpp +++ b/src/core/main.cpp @@ -983,6 +983,7 @@ static void do_global_ctors_helper() { static lock_spin_recursive g_globals_lock; std::lock_guard lock(g_globals_lock); + int rc; if (g_init_global_ctors_done) { return; @@ -1018,6 +1019,12 @@ static void do_global_ctors_helper() // Create all global management objects NEW_CTOR(g_p_event_handler_manager, event_handler_manager()); + rc = g_p_event_handler_manager->start_thread(); + if (rc != 0) { + BULLSEYE_EXCLUDE_BLOCK_START + throw_xlio_exception("Failed to start event handler thread.\n"); + BULLSEYE_EXCLUDE_BLOCK_END + } xlio_shmem_stats_open(&g_p_vlogger_level, &g_p_vlogger_details); *g_p_vlogger_level = g_vlogger_level;