diff --git a/configure.ac b/configure.ac index 16351e0..1b07cca 100644 --- a/configure.ac +++ b/configure.ac @@ -19,6 +19,11 @@ AC_SEARCH_LIBS([rdma_create_id], [rdmacm], [], [ ]) AC_CHECK_HEADERS([rdma/rdma_cma.h],[], [AC_MSG_ERROR(missing rdma headers)]) +# Check if RDMA_CM Libs supports rdma_lib_reset() +AC_CHECK_FUNC([rdma_lib_reset], [ + AC_DEFINE([HAVE_RDMA_LIB_RESET], [1], [Define if rdma_lib_reset() is supported.]) +]) + AC_DEFINE( [VERSION_COMMENT], ["libmooshika and examples"], [No Comment]) # Git latest commit diff --git a/include/mooshika.h b/include/mooshika.h index f606491..c2cfeb1 100644 --- a/include/mooshika.h +++ b/include/mooshika.h @@ -226,6 +226,9 @@ static inline int msk_wait_write(msk_trans_t *trans, msk_data_t *data, msk_rloc_ int msk_init(msk_trans_t **ptrans, msk_trans_attr_t *attr); +void msk_lib_reset(void); +int msk_fork_init(void); + // server specific: int msk_bind_server(msk_trans_t *trans); msk_trans_t *msk_accept_one_wait(msk_trans_t *trans, int msleep); diff --git a/src/trans_rdma.c b/src/trans_rdma.c index 4c3f356..531cf51 100644 --- a/src/trans_rdma.c +++ b/src/trans_rdma.c @@ -175,6 +175,48 @@ void __attribute__ ((destructor)) msk_internals_fini(void) { } } +/** + * Reset global state. + * Should (only) be called to reset global data in child process after a fork. + */ +void msk_lib_reset(void) +{ + /* Close epoll fd's */ + if (msk_global_state->cm_epollfd != 0) + close(msk_global_state->cm_epollfd); + + if (msk_global_state->cq_epollfd != 0) + close(msk_global_state->cq_epollfd); + + if (msk_global_state->stats_epollfd != 0) + close(msk_global_state->stats_epollfd); + + /* Free worker pool resources */ + if (msk_global_state->worker_pool.w_efd != 0) + close(msk_global_state->worker_pool.w_efd); + + if (msk_global_state->worker_pool.m_efd != 0) + close(msk_global_state->worker_pool.m_efd); + + if (msk_global_state->worker_pool.thrids) + free(msk_global_state->worker_pool.thrids); + + if (msk_global_state->worker_pool.wd_queue) + free(msk_global_state->worker_pool.wd_queue); + + /* Brutal re-initialization of global state */ + memset(msk_global_state, 0, sizeof(*msk_global_state)); + + msk_global_state->run_threads = 0; + if (pthread_mutex_init(&msk_global_state->lock, NULL)) + ERROR_LOG("pthread_mutex_init failed?!"); + +#ifdef HAVE_RDMA_LIB_RESET + /* Reset librdmacm (Mellanox MOFED interface) */ + rdma_lib_reset(); +#endif +} + /* forward declarations */ static void *msk_cq_thread(void *arg); @@ -212,6 +254,17 @@ static inline int msk_cond_timedwait(int debug, } +/** + * msk_fork_init: wrapper around ibv_fork_init() + * Initialize libibverbs to support fork(). + * + * @return 0 on success, the value of errno on failure + */ +int msk_fork_init(void) +{ + return ibv_fork_init(); +} + /** * msk_getpd: helper function to get the right pd for a given trans *