From: Amir Shehata Date: Wed, 24 May 2017 02:44:21 +0000 (-0700) Subject: LU-9448 lnet: handle empty CPTs X-Git-Tag: 2.9.59~63 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=e711370e13dcbe059e2551aa575c41d62cbcfca9;p=fs%2Flustre-release.git LU-9448 lnet: handle empty CPTs Make the LND code handle empty CPTs. If a scheduler is associated with an empty CPT it will have no threads created. If a NID hashes to that CPT, then pick the next scheduler which does have at least 1 started thread. Associate the connection with the CPT of the selected scheduler Test-Parameters: trivial Signed-off-by: Amir Shehata Change-Id: I8ed83f295fe9852537d4bb063a4d8271c6a45c2c Reviewed-on: https://review.whamcloud.com/27145 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Doug Oucharek Reviewed-by: Dmitry Eremin Reviewed-by: Olaf Weber Reviewed-by: Oleg Drokin --- diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c index 70ed74f..12dade5 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.c +++ b/lnet/klnds/o2iblnd/o2iblnd.c @@ -698,6 +698,36 @@ kiblnd_get_completion_vector(kib_conn_t *conn, int cpt) return 1; } +/* + * Get the scheduler bound to this CPT. If the scheduler has no + * threads, which means that the CPT has no CPUs, then grab the + * next scheduler that we can use. + * + * This case would be triggered if a NUMA node is configured with + * no associated CPUs. + */ +static struct kib_sched_info * +kiblnd_get_scheduler(int cpt) +{ + struct kib_sched_info *sched; + int i; + + sched = kiblnd_data.kib_scheds[cpt]; + + if (sched->ibs_nthreads > 0) + return sched; + + cfs_percpt_for_each(sched, i, kiblnd_data.kib_scheds) { + if (sched->ibs_nthreads > 0) { + CDEBUG(D_NET, "scheduler[%d] has no threads. selected scheduler[%d]\n", + cpt, sched->ibs_cpt); + return sched; + } + } + + return NULL; +} + kib_conn_t * kiblnd_create_conn(kib_peer_ni_t *peer_ni, struct rdma_cm_id *cmid, int state, int version) @@ -730,9 +760,18 @@ kiblnd_create_conn(kib_peer_ni_t *peer_ni, struct rdma_cm_id *cmid, dev = net->ibn_dev; cpt = lnet_cpt_of_nid(peer_ni->ibp_nid, peer_ni->ibp_ni); - sched = kiblnd_data.kib_scheds[cpt]; + sched = kiblnd_get_scheduler(cpt); + + if (sched == NULL) { + CERROR("no schedulers available. node is unhealthy\n"); + goto failed_0; + } - LASSERT(sched->ibs_nthreads > 0); + /* + * The cpt might have changed if we ended up selecting a non cpt + * native scheduler. So use the scheduler's cpt instead. + */ + cpt = sched->ibs_cpt; LIBCFS_CPT_ALLOC(init_qp_attr, lnet_cpt_table(), cpt, sizeof(*init_qp_attr)); diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c index 0852e85..0fdedba 100644 --- a/lnet/klnds/socklnd/socklnd.c +++ b/lnet/klnds/socklnd/socklnd.c @@ -670,8 +670,18 @@ ksocknal_choose_scheduler_locked(unsigned int cpt) ksock_sched_t *sched; int i; - LASSERT(info->ksi_nthreads > 0); + if (info->ksi_nthreads == 0) { + cfs_percpt_for_each(info, i, ksocknal_data.ksnd_sched_info) { + if (info->ksi_nthreads > 0) { + CDEBUG(D_NET, "scheduler[%d] has no threads. selected scheduler[%d]\n", + cpt, info->ksi_cpt); + goto select_sched; + } + } + return NULL; + } +select_sched: sched = &info->ksi_scheds[0]; /* * NB: it's safe so far, but info->ksi_nthreads could be changed @@ -1263,6 +1273,15 @@ ksocknal_create_conn(struct lnet_ni *ni, ksock_route_t *route, peer_ni->ksnp_error = 0; sched = ksocknal_choose_scheduler_locked(cpt); + if (!sched) { + CERROR("no schedulers available. node is unhealthy\n"); + goto failed_2; + } + /* + * The cpt might have changed if we ended up selecting a non cpt + * native scheduler. So use the scheduler's cpt instead. + */ + cpt = sched->kss_info->ksi_cpt; sched->kss_nconns++; conn->ksnc_scheduler = sched; @@ -2425,20 +2444,23 @@ ksocknal_base_startup(void) info->ksi_nthreads_max = nthrs; info->ksi_cpt = i; - LIBCFS_CPT_ALLOC(info->ksi_scheds, lnet_cpt_table(), i, - info->ksi_nthreads_max * sizeof(*sched)); - if (info->ksi_scheds == NULL) - goto failed; - - for (; nthrs > 0; nthrs--) { - sched = &info->ksi_scheds[nthrs - 1]; - - sched->kss_info = info; - spin_lock_init(&sched->kss_lock); - INIT_LIST_HEAD(&sched->kss_rx_conns); - INIT_LIST_HEAD(&sched->kss_tx_conns); - INIT_LIST_HEAD(&sched->kss_zombie_noop_txs); - init_waitqueue_head(&sched->kss_waitq); + if (nthrs != 0) { + LIBCFS_CPT_ALLOC(info->ksi_scheds, lnet_cpt_table(), i, + info->ksi_nthreads_max * + sizeof(*sched)); + if (info->ksi_scheds == NULL) + goto failed; + + for (; nthrs > 0; nthrs--) { + sched = &info->ksi_scheds[nthrs - 1]; + + sched->kss_info = info; + spin_lock_init(&sched->kss_lock); + INIT_LIST_HEAD(&sched->kss_rx_conns); + INIT_LIST_HEAD(&sched->kss_tx_conns); + INIT_LIST_HEAD(&sched->kss_zombie_noop_txs); + init_waitqueue_head(&sched->kss_waitq); + } } }