/*
* Copyright (C) 2012 Cray, Inc.
*
+ * Copyright (c) 2013, 2014, Intel Corporation.
+ *
* Author: Nic Henke <nic@cray.com>
* Author: James Shimek <jshimek@cray.com>
*
/* Primary entry points from LNET. There are no guarantees against reentrance. */
lnd_t the_kgnilnd = {
+#ifdef CONFIG_CRAY_XT
.lnd_type = GNILND,
+#else
+ .lnd_type = GNIIPLND,
+#endif
.lnd_startup = kgnilnd_startup,
.lnd_shutdown = kgnilnd_shutdown,
.lnd_ctl = kgnilnd_ctl,
kgn_data_t kgnilnd_data;
+int
+kgnilnd_thread_start(int(*fn)(void *arg), void *arg, char *name, int id)
+{
+ struct task_struct *thrd;
+
+ thrd = kthread_run(fn, arg, "%s_%02d", name, id);
+ if (IS_ERR(thrd))
+ return PTR_ERR(thrd);
+
+ atomic_inc(&kgnilnd_data.kgn_nthreads);
+ return 0;
+}
+
+/* bind scheduler threads to cpus */
+int
+kgnilnd_start_sd_threads(void)
+{
+ int cpu;
+ int i = 0;
+ struct task_struct *task;
+
+ for_each_online_cpu(cpu) {
+ /* don't bind to cpu 0 - all interrupts are processed here */
+ if (cpu == 0)
+ continue;
+
+ task = kthread_create(kgnilnd_scheduler, (void *)((long)i),
+ "%s_%02d", "kgnilnd_sd", i);
+ if (!IS_ERR(task)) {
+ kthread_bind(task, cpu);
+ wake_up_process(task);
+ } else {
+ CERROR("Can't spawn gnilnd scheduler[%d] %ld\n", i,
+ PTR_ERR(task));
+ return PTR_ERR(task);
+ }
+ atomic_inc(&kgnilnd_data.kgn_nthreads);
+
+ if (++i >= *kgnilnd_tunables.kgn_sched_threads) {
+ break;
+ }
+ }
+
+ return 0;
+}
+
/* needs write_lock on kgn_peer_conn_lock */
int
kgnilnd_close_stale_conns_locked(kgn_peer_t *peer, kgn_conn_t *newconn)
newconn->gnc_peer_connstamp == conn->gnc_my_connstamp) {
CDEBUG(D_NET, "skipping prune of %p, "
"loopback and matching stamps"
- " connstamp "LPU64"("LPU64")"
- " peerstamp "LPU64"("LPU64")\n",
+ " connstamp %llu(%llu)"
+ " peerstamp %llu(%llu)\n",
conn, newconn->gnc_my_connstamp,
conn->gnc_peer_connstamp,
newconn->gnc_peer_connstamp,
if (conn->gnc_peerstamp != newconn->gnc_peerstamp) {
LASSERTF(conn->gnc_peerstamp < newconn->gnc_peerstamp,
- "conn 0x%p peerstamp "LPU64" >= "
- "newconn 0x%p peerstamp "LPU64"\n",
+ "conn 0x%p peerstamp %llu >= "
+ "newconn 0x%p peerstamp %llu\n",
conn, conn->gnc_peerstamp,
newconn, newconn->gnc_peerstamp);
CDEBUG(D_NET, "Closing stale conn nid: %s "
- " peerstamp:"LPX64"("LPX64")\n",
+ " peerstamp:%#llx(%#llx)\n",
libcfs_nid2str(peer->gnp_nid),
conn->gnc_peerstamp, newconn->gnc_peerstamp);
} else {
LASSERTF(conn->gnc_peer_connstamp < newconn->gnc_peer_connstamp,
- "conn 0x%p peer_connstamp "LPU64" >= "
- "newconn 0x%p peer_connstamp "LPU64"\n",
+ "conn 0x%p peer_connstamp %llu >= "
+ "newconn 0x%p peer_connstamp %llu\n",
conn, conn->gnc_peer_connstamp,
newconn, newconn->gnc_peer_connstamp);
CDEBUG(D_NET, "Closing stale conn nid: %s"
- " connstamp:"LPU64"("LPU64")\n",
+ " connstamp:%llu(%llu)\n",
libcfs_nid2str(peer->gnp_nid),
conn->gnc_peer_connstamp, newconn->gnc_peer_connstamp);
}
list_for_each(tmp, &peer->gnp_conns) {
conn = list_entry(tmp, kgn_conn_t, gnc_list);
CDEBUG(D_NET, "checking conn 0x%p for peer %s"
- " lo %d new "LPU64" existing "LPU64
- " new peer "LPU64" existing peer "LPU64
+ " lo %d new %llu existing %llu"
+ " new peer %llu existing peer %llu"
" new dev %p existing dev %p\n",
conn, libcfs_nid2str(peer->gnp_nid),
loopback,
return -ENOMEM;
}
- LIBCFS_ALLOC(conn->gnc_tx_ref_table, GNILND_MAX_MSG_ID * sizeof(void *));
+ conn->gnc_tx_ref_table =
+ kgnilnd_vzalloc(GNILND_MAX_MSG_ID * sizeof(void *));
if (conn->gnc_tx_ref_table == NULL) {
CERROR("Can't allocate conn tx_ref_table\n");
- rc = -ENOMEM;
- GOTO(failed, rc);
+ GOTO(failed, rc = -ENOMEM);
}
+ mutex_init(&conn->gnc_smsg_mutex);
+ mutex_init(&conn->gnc_rdma_mutex);
atomic_set(&conn->gnc_refcount, 1);
atomic_set(&conn->gnc_reaper_noop, 0);
atomic_set(&conn->gnc_sched_noop, 0);
if (conn->gnc_cqid == 0) {
CERROR("Could not allocate unique CQ ID for conn 0x%p\n", conn);
- rc = -E2BIG;
- GOTO(failed, rc);
+ GOTO(failed, rc = -E2BIG);
}
CDEBUG(D_NET, "alloc cqid %u for conn 0x%p\n",
rrc = kgnilnd_ep_create(dev->gnd_handle, dev->gnd_snd_fma_cqh,
&conn->gnc_ephandle);
mutex_unlock(&dev->gnd_cq_mutex);
- if (rrc != GNI_RC_SUCCESS) {
- rc = -ENETDOWN;
- GOTO(failed, rc);
- }
+ if (rrc != GNI_RC_SUCCESS)
+ GOTO(failed, rc = -ENETDOWN);
CDEBUG(D_NET, "created conn 0x%p ep_hndl 0x%p\n",
conn, conn->gnc_ephandle);
failed:
atomic_dec(&kgnilnd_data.kgn_nconns);
- LIBCFS_FREE(conn->gnc_tx_ref_table, GNILND_MAX_MSG_ID * sizeof(void *));
+ kgnilnd_vfree(conn->gnc_tx_ref_table,
+ GNILND_MAX_MSG_ID * sizeof(void *));
LIBCFS_FREE(conn, sizeof(*conn));
return rc;
}
kgnilnd_peer_decref(conn->gnc_peer);
if (conn->gnc_tx_ref_table != NULL) {
- LIBCFS_FREE(conn->gnc_tx_ref_table,
- GNILND_MAX_MSG_ID * sizeof(void *));
+ kgnilnd_vfree(conn->gnc_tx_ref_table,
+ GNILND_MAX_MSG_ID * sizeof(void *));
}
LIBCFS_FREE(conn, sizeof(*conn));
}
void
-kgnilnd_peer_notify(kgn_peer_t *peer, int error)
+kgnilnd_peer_notify(kgn_peer_t *peer, int error, int alive)
{
int tell_lnet = 0;
int nnets = 0;
peer, libcfs_nid2str(peer->gnp_nid), peer->gnp_connecting, conn,
kgnilnd_data.kgn_in_reset, error);
- if ((peer->gnp_connecting == GNILND_PEER_IDLE) &&
+ if (((peer->gnp_connecting == GNILND_PEER_IDLE) &&
(conn == NULL) &&
(!kgnilnd_data.kgn_in_reset) &&
- (!kgnilnd_conn_clean_errno(error))) {
+ (!kgnilnd_conn_clean_errno(error))) || alive) {
tell_lnet = 1;
}
peer, libcfs_nid2str(peer_nid), peer->gnp_last_alive,
cfs_duration_sec(jiffies - peer->gnp_last_alive));
- lnet_notify(net->gnn_ni, peer_nid, 0, peer->gnp_last_alive);
-
+ lnet_notify(net->gnn_ni, peer_nid, alive,
+ peer->gnp_last_alive);
kgnilnd_net_decref(net);
}
conn->gnc_state = GNILND_CONN_CLOSING;
}
+ if (CFS_FAIL_CHECK(CFS_FAIL_GNI_RDMA_CQ_ERROR)) {
+ msleep_interruptible(MSEC_PER_SEC);
+ }
+
/* leave on peer->gnp_conns to make sure we don't let the reaper
* or others try to unlink this peer until the conn is fully
* processed for closing */
/* I'm telling Mommy! - use peer_error if they initiated close */
kgnilnd_peer_notify(conn->gnc_peer,
- conn->gnc_error == -ECONNRESET ? conn->gnc_peer_error
- : conn->gnc_error);
+ conn->gnc_error == -ECONNRESET ?
+ conn->gnc_peer_error : conn->gnc_error, 0);
EXIT;
}
/* log this for help in debuggin SMSG buffer re-use */
CDEBUG(D_NET, "conn %p src %s dst %s smsg %p acquired"
- " local cqid %u SMSG %p->%u hndl "LPX64"."LPX64
- " remote cqid %u SMSG %p->%u hndl "LPX64"."LPX64"\n",
+ " local cqid %u SMSG %p->%u hndl %#llx.%#llx"
+ " remote cqid %u SMSG %p->%u hndl %#llx.%#llx\n",
conn, libcfs_nid2str(connreq->gncr_srcnid),
libcfs_nid2str(connreq->gncr_dstnid),
&conn->gnpr_smsg_attr,
* kgn_peer_conn_lock is held, we guarantee that nobody calls
* kgnilnd_add_peer_locked without checking gnn_shutdown */
int
-kgnilnd_create_peer_safe(kgn_peer_t **peerp, lnet_nid_t nid, kgn_net_t *net)
+kgnilnd_create_peer_safe(kgn_peer_t **peerp,
+ lnet_nid_t nid,
+ kgn_net_t *net,
+ int node_state)
{
kgn_peer_t *peer;
int rc;
return -ENOMEM;
}
peer->gnp_nid = nid;
- peer->gnp_down = GNILND_RCA_NODE_UP;
+ peer->gnp_down = node_state;
/* translate from nid to nic addr & store */
rc = kgnilnd_nid_to_nicaddrs(LNET_NIDADDR(nid), 1, &peer->gnp_host_id);
CDEBUG(D_NET, "conn %p peer %p dev %p\n", conn, peer,
conn->gnc_device);
+ LASSERTF(conn->gnc_in_purgatory == 0,
+ "Conn already in purgatory\n");
conn->gnc_in_purgatory = 1;
mbox = &conn->gnc_fma_blk->gnm_mbox_info[conn->gnc_mbox_id];
* make sure we tell LNet - if this is from other context,
* the checks in the function will prevent an errant
* notification */
- kgnilnd_peer_notify(conn->gnc_peer, conn->gnc_error);
+ kgnilnd_peer_notify(conn->gnc_peer, conn->gnc_error, 0);
list_for_each_entry_safe(gmp, gmpN, &conn->gnc_mdd_list,
gmp_list) {
CDEBUG(D_NET,
- "dev %p releasing held mdd "LPX64"."LPX64"\n",
+ "dev %p releasing held mdd %#llx.%#llx\n",
conn->gnc_device, gmp->gmp_map_key.qword1,
gmp->gmp_map_key.qword2);
{
kgn_peer_t *peer;
int rc;
+ int node_state;
ENTRY;
if (nid == LNET_NID_ANY)
return -EINVAL;
+ node_state = kgnilnd_get_node_state(LNET_NIDADDR(nid));
+
/* NB - this will not block during normal operations -
* the only writer of this is in the startup/shutdown path. */
rc = down_read_trylock(&kgnilnd_data.kgn_net_rw_sem);
rc = -ESHUTDOWN;
RETURN(rc);
}
- rc = kgnilnd_create_peer_safe(&peer, nid, net);
+ rc = kgnilnd_create_peer_safe(&peer, nid, net, node_state);
if (rc != 0) {
up_read(&kgnilnd_data.kgn_net_rw_sem);
RETURN(rc);
write_unlock(&kgnilnd_data.kgn_peer_conn_lock);
- /* release all of the souls found held in purgatory */
- kgnilnd_release_purgatory_list(&souls);
-
/* nuke peer TX */
kgnilnd_txlist_done(&zombies, error);
atomic_read(&kgnilnd_data.kgn_npending_detach) ||
atomic_read(&kgnilnd_data.kgn_npending_unlink)) {
- cfs_pause(cfs_time_seconds(1));
+ set_current_state(TASK_UNINTERRUPTIBLE);
+ schedule_timeout(cfs_time_seconds(1));
i++;
CDEBUG(((i & (-i)) == i) ? D_WARNING : D_NET, "Waiting on %d peers %d closes %d detaches\n",
*device_id = conn->gnc_device->gnd_host_id;
*peerstamp = conn->gnc_peerstamp;
- *tx_seq = conn->gnc_tx_seq;
- *rx_seq = conn->gnc_rx_seq;
+ *tx_seq = atomic_read(&conn->gnc_tx_seq);
+ *rx_seq = atomic_read(&conn->gnc_rx_seq);
*fmaq_len = kgnilnd_count_list(&conn->gnc_fmaq);
*nfma = atomic_read(&conn->gnc_nlive_fma);
*nrdma = atomic_read(&conn->gnc_nlive_rdma);
{
int rc;
kgn_peer_t *peer, *new_peer;
- CFS_LIST_HEAD(zombies);
+ LIST_HEAD(zombies);
write_lock(&kgnilnd_data.kgn_peer_conn_lock);
peer = kgnilnd_find_peer_locked(nid);
* kgnilnd_tx_done
*/
kgnilnd_txlist_done(&zombies, -ENETRESET);
-
- if (*kgnilnd_tunables.kgn_peer_health) {
- kgnilnd_peer_notify(peer, -ECONNRESET);
- }
+ kgnilnd_peer_notify(peer, -ECONNRESET, 0);
+ LCONSOLE_INFO("Received down event for nid %d\n",
+ LNET_NIDADDR(nid));
}
- CDEBUG(D_INFO, "marking nid %lld %s\n", nid, down ? "down" : "up");
return 0;
}
kgn_tx_t *tx;
kgn_peer_t *peer = NULL;
kgn_conn_t *conn = NULL;
- lnet_process_id_t id = {.nid = nid, .pid = LUSTRE_SRV_LNET_PID};
+ lnet_process_id_t id = {
+ .nid = nid,
+ .pid = LNET_PID_LUSTRE,
+ };
ENTRY;
/* I expect to find him, so only take a read lock */
cq_size = *kgnilnd_tunables.kgn_credits * 2 * 3;
rrc = kgnilnd_cdm_create(dev->gnd_id, *kgnilnd_tunables.kgn_ptag,
- GNILND_COOKIE, 0,
+ *kgnilnd_tunables.kgn_pkey, 0,
&dev->gnd_domain);
if (rrc != GNI_RC_SUCCESS) {
CERROR("Can't create CDM %d (%d)\n", dev->gnd_id, rrc);
- rc = -ENODEV;
- GOTO(failed, rc);
+ GOTO(failed, rc = -ENODEV);
}
rrc = kgnilnd_cdm_attach(dev->gnd_domain, dev->gnd_id,
if (rrc != GNI_RC_SUCCESS) {
CERROR("Can't attach CDM to device %d (%d)\n",
dev->gnd_id, rrc);
- rc = -ENODEV;
- GOTO(failed, rc);
+ GOTO(failed, rc = -ENODEV);
}
/* a bit gross, but not much we can do - Aries Sim doesn't have
* hardcoded NIC/NID that we can use */
rc = kgnilnd_setup_nic_translation(dev->gnd_host_id);
- if (rc != 0) {
- rc = -ENODEV;
- GOTO(failed, rc);
- }
+ if (rc != 0)
+ GOTO(failed, rc = -ENODEV);
/* only dev 0 gets the errors - no need to reset the stack twice
* - this works because we have a single PTAG, if we had more
if (rrc != GNI_RC_SUCCESS) {
CERROR("Can't subscribe for errors on device %d: rc %d\n",
dev->gnd_id, rrc);
- rc = -ENODEV;
- GOTO(failed, rc);
+ GOTO(failed, rc = -ENODEV);
}
rc = kgnilnd_set_quiesce_callback(dev->gnd_handle,
if (rc != GNI_RC_SUCCESS) {
CERROR("Can't subscribe for quiesce callback on device %d: rc %d\n",
dev->gnd_id, rrc);
- rc = -ENODEV;
- GOTO(failed, rc);
+ GOTO(failed, rc = -ENODEV);
}
}
+ rrc = sock_create_kern(PF_INET, SOCK_DGRAM, IPPROTO_IP, &kgnilnd_data.kgn_sock);
+ if (rrc < 0) {
+ CERROR("sock_create returned %d\n", rrc);
+ GOTO(failed, rrc);
+ }
+
rc = kgnilnd_nicaddr_to_nid(dev->gnd_host_id, &dev->gnd_nid);
if (rc < 0) {
/* log messages during startup */
CERROR("couldn't translate host_id 0x%x to nid. rc %d\n",
dev->gnd_host_id, rc);
}
- rc = -ESRCH;
- GOTO(failed, rc);
+ GOTO(failed, rc = -ESRCH);
}
CDEBUG(D_NET, "NIC %x -> NID %d\n", dev->gnd_host_id, dev->gnd_nid);
- rrc = kgnilnd_cq_create(dev->gnd_handle, cq_size,
+ rrc = kgnilnd_cq_create(dev->gnd_handle, *kgnilnd_tunables.kgn_credits,
0, kgnilnd_device_callback,
dev->gnd_id, &dev->gnd_snd_rdma_cqh);
if (rrc != GNI_RC_SUCCESS) {
CERROR("Can't create rdma send cq size %u for device "
"%d (%d)\n", cq_size, dev->gnd_id, rrc);
- rc = -EINVAL;
- GOTO(failed, rc);
+ GOTO(failed, rc = -EINVAL);
}
rrc = kgnilnd_cq_create(dev->gnd_handle, cq_size,
if (rrc != GNI_RC_SUCCESS) {
CERROR("Can't create fma send cq size %u for device %d (%d)\n",
cq_size, dev->gnd_id, rrc);
- rc = -EINVAL;
- GOTO(failed, rc);
+ GOTO(failed, rc = -EINVAL);
}
/* This one we size differently - overflows are possible and it needs to be
if (rrc != GNI_RC_SUCCESS) {
CERROR("Can't create fma cq size %d for device %d (%d)\n",
*kgnilnd_tunables.kgn_fma_cq_size, dev->gnd_id, rrc);
- rc = -EINVAL;
- GOTO(failed, rc);
+ GOTO(failed, rc = -EINVAL);
+ }
+
+ rrc = kgnilnd_register_smdd_buf(dev);
+ if (rrc != GNI_RC_SUCCESS) {
+ GOTO(failed, rc = -EINVAL);
}
RETURN(0);
dev->gnd_map_nphys, dev->gnd_map_physnop);
LASSERTF(dev->gnd_map_nvirt == 0 && dev->gnd_map_virtnob == 0,
- "%d virtual mappings of "LPU64" bytes still mapped\n",
+ "%d virtual mappings of %llu bytes still mapped\n",
dev->gnd_map_nvirt, dev->gnd_map_virtnob);
LASSERTF(atomic_read(&dev->gnd_n_mdd) == 0 &&
atomic_read(&kgnilnd_data.kgn_nthreads) == 0,
"tried to shutdown with threads active\n");
+ if (dev->gnd_smdd_hold_buf) {
+ rrc = kgnilnd_deregister_smdd_buf(dev);
+ LASSERTF(rrc == GNI_RC_SUCCESS,
+ "bad rc from deregistion of sMDD buffer: %d\n", rrc);
+ dev->gnd_smdd_hold_buf = NULL;
+ }
+
if (dev->gnd_rcv_fma_cqh) {
rrc = kgnilnd_cq_destroy(dev->gnd_rcv_fma_cqh);
LASSERTF(rrc == GNI_RC_SUCCESS,
dev->gnd_domain = NULL;
}
+ if (kgnilnd_data.kgn_sock)
+ sock_release(kgnilnd_data.kgn_sock);
+
EXIT;
}
-
int kgnilnd_base_startup(void)
{
struct timeval tv;
int i;
kgn_device_t *dev;
struct task_struct *thrd;
+
+#if defined(CONFIG_CRAY_XT) && !defined(CONFIG_CRAY_COMPUTE)
+ /* limit how much memory can be allocated for fma blocks in
+ * instances where many nodes need to reconnects at the same time */
+ struct sysinfo si;
+ si_meminfo(&si);
+ kgnilnd_data.free_pages_limit = si.totalram/4;
+#endif
+
ENTRY;
LASSERTF(kgnilnd_data.kgn_init == GNILND_INIT_NOTHING,
/* zero pointers, flags etc */
memset(&kgnilnd_data, 0, sizeof(kgnilnd_data));
+ kgnilnd_check_kgni_version();
/* CAVEAT EMPTOR: Every 'Fma' message includes the sender's NID and
* a unique (for all time) connstamp so we can uniquely identify
INIT_LIST_HEAD(&dev->gnd_map_tx);
INIT_LIST_HEAD(&dev->gnd_fma_buffs);
mutex_init(&dev->gnd_cq_mutex);
- sema_init(&dev->gnd_fmablk_sem, 1);
+ mutex_init(&dev->gnd_fmablk_mutex);
spin_lock_init(&dev->gnd_fmablk_lock);
init_waitqueue_head(&dev->gnd_waitq);
init_waitqueue_head(&dev->gnd_dgram_waitq);
LIBCFS_ALLOC(dev->gnd_dgrams,
sizeof(struct list_head) * *kgnilnd_tunables.kgn_peer_hash_size);
- if (dev->gnd_dgrams == NULL) {
- rc = -ENOMEM;
- GOTO(failed, rc);
- }
+ if (dev->gnd_dgrams == NULL)
+ GOTO(failed, rc = -ENOMEM);
for (i = 0; i < *kgnilnd_tunables.kgn_peer_hash_size; i++) {
INIT_LIST_HEAD(&dev->gnd_dgrams[i]);
init_waitqueue_head(&kgnilnd_data.kgn_ruhroh_waitq);
spin_lock_init(&kgnilnd_data.kgn_reaper_lock);
- sema_init(&kgnilnd_data.kgn_quiesce_sem, 1);
+ mutex_init(&kgnilnd_data.kgn_quiesce_mutex);
atomic_set(&kgnilnd_data.kgn_nquiesce, 0);
atomic_set(&kgnilnd_data.kgn_npending_conns, 0);
atomic_set(&kgnilnd_data.kgn_npending_unlink, 0);
/* OK to call kgnilnd_api_shutdown() to cleanup now */
kgnilnd_data.kgn_init = GNILND_INIT_DATA;
- PORTAL_MODULE_USE;
+ try_module_get(THIS_MODULE);
rwlock_init(&kgnilnd_data.kgn_peer_conn_lock);
LIBCFS_ALLOC(kgnilnd_data.kgn_peers,
sizeof(struct list_head) * *kgnilnd_tunables.kgn_peer_hash_size);
- if (kgnilnd_data.kgn_peers == NULL) {
- rc = -ENOMEM;
- GOTO(failed, rc);
- }
+ if (kgnilnd_data.kgn_peers == NULL)
+ GOTO(failed, rc = -ENOMEM);
for (i = 0; i < *kgnilnd_tunables.kgn_peer_hash_size; i++) {
INIT_LIST_HEAD(&kgnilnd_data.kgn_peers[i]);
LIBCFS_ALLOC(kgnilnd_data.kgn_conns,
sizeof(struct list_head) * *kgnilnd_tunables.kgn_peer_hash_size);
- if (kgnilnd_data.kgn_conns == NULL) {
- rc = -ENOMEM;
- GOTO(failed, rc);
- }
+ if (kgnilnd_data.kgn_conns == NULL)
+ GOTO(failed, rc = -ENOMEM);
for (i = 0; i < *kgnilnd_tunables.kgn_peer_hash_size; i++) {
INIT_LIST_HEAD(&kgnilnd_data.kgn_conns[i]);
LIBCFS_ALLOC(kgnilnd_data.kgn_nets,
sizeof(struct list_head) * *kgnilnd_tunables.kgn_net_hash_size);
- if (kgnilnd_data.kgn_nets == NULL) {
- rc = -ENOMEM;
- GOTO(failed, rc);
- }
+ if (kgnilnd_data.kgn_nets == NULL)
+ GOTO(failed, rc = -ENOMEM);
for (i = 0; i < *kgnilnd_tunables.kgn_net_hash_size; i++) {
INIT_LIST_HEAD(&kgnilnd_data.kgn_nets[i]);
}
kgnilnd_data.kgn_mbox_cache =
- cfs_mem_cache_create("kgn_mbox_block",
- KMALLOC_MAX_SIZE,
- 0, /* offset */
- SLAB_HWCACHE_ALIGN); /* flags */
+ kmem_cache_create("kgn_mbox_block", KMALLOC_MAX_SIZE, 0,
+ SLAB_HWCACHE_ALIGN, NULL);
if (kgnilnd_data.kgn_mbox_cache == NULL) {
CERROR("Can't create slab for physical mbox blocks\n");
- rc = -ENOMEM;
- GOTO(failed, rc);
+ GOTO(failed, rc = -ENOMEM);
}
kgnilnd_data.kgn_rx_cache =
- cfs_mem_cache_create("kgn_rx_t",
- sizeof(kgn_rx_t),
- 0, /* offset */
- 0); /* flags */
+ kmem_cache_create("kgn_rx_t", sizeof(kgn_rx_t), 0, 0, NULL);
if (kgnilnd_data.kgn_rx_cache == NULL) {
CERROR("Can't create slab for kgn_rx_t descriptors\n");
- rc = -ENOMEM;
- GOTO(failed, rc);
+ GOTO(failed, rc = -ENOMEM);
}
kgnilnd_data.kgn_tx_cache =
- cfs_mem_cache_create("kgn_tx_t",
- sizeof(kgn_tx_t),
- 0, /* offset */
- 0); /* flags */
+ kmem_cache_create("kgn_tx_t", sizeof(kgn_tx_t), 0, 0, NULL);
if (kgnilnd_data.kgn_tx_cache == NULL) {
CERROR("Can't create slab for kgn_tx_t\n");
- rc = -ENOMEM;
- GOTO(failed, rc);
+ GOTO(failed, rc = -ENOMEM);
}
kgnilnd_data.kgn_tx_phys_cache =
- cfs_mem_cache_create("kgn_tx_phys",
- LNET_MAX_IOV * sizeof(gni_mem_segment_t),
- 0, /* offset */
- 0); /* flags */
+ kmem_cache_create("kgn_tx_phys",
+ LNET_MAX_IOV * sizeof(gni_mem_segment_t),
+ 0, 0, NULL);
if (kgnilnd_data.kgn_tx_phys_cache == NULL) {
CERROR("Can't create slab for kgn_tx_phys\n");
- rc = -ENOMEM;
- GOTO(failed, rc);
+ GOTO(failed, rc = -ENOMEM);
}
kgnilnd_data.kgn_dgram_cache =
- cfs_mem_cache_create("kgn_dgram_t",
- sizeof(kgn_dgram_t),
- 0, /* offset */
- 0); /* flags */
+ kmem_cache_create("kgn_dgram_t", sizeof(kgn_dgram_t), 0, 0, NULL);
if (kgnilnd_data.kgn_dgram_cache == NULL) {
CERROR("Can't create slab for outgoing datagrams\n");
- rc = -ENOMEM;
- GOTO(failed, rc);
+ GOTO(failed, rc = -ENOMEM);
}
/* allocate a MAX_IOV array of page pointers for each cpu */
GFP_KERNEL);
if (kgnilnd_data.kgn_cksum_map_pages == NULL) {
CERROR("Can't allocate vmap cksum pages\n");
- rc = -ENOMEM;
- GOTO(failed, rc);
+ GOTO(failed, rc = -ENOMEM);
}
kgnilnd_data.kgn_cksum_npages = num_possible_cpus();
memset(kgnilnd_data.kgn_cksum_map_pages, 0,
GFP_KERNEL);
if (kgnilnd_data.kgn_cksum_map_pages[i] == NULL) {
CERROR("Can't allocate vmap cksum pages for cpu %d\n", i);
- rc = -ENOMEM;
- GOTO(failed, rc);
+ GOTO(failed, rc = -ENOMEM);
}
}
kgnilnd_data.kgn_ndevs++;
rc = kgnilnd_allocate_phys_fmablk(dev);
- if (rc) {
+ if (rc)
GOTO(failed, rc);
- }
}
}
if (kgnilnd_data.kgn_ndevs == 0) {
CERROR("Can't initialise any GNI devices\n");
- rc = -ENODEV;
- GOTO(failed, rc);
+ GOTO(failed, rc = -ENODEV);
}
rc = kgnilnd_thread_start(kgnilnd_reaper, NULL, "kgnilnd_rpr", 0);
}
/* threads will load balance across devs as they are available */
- for (i = 0; i < *kgnilnd_tunables.kgn_sched_threads; i++) {
- rc = kgnilnd_thread_start(kgnilnd_scheduler, (void *)((long)i),
- "kgnilnd_sd", i);
- if (rc != 0) {
- CERROR("Can't spawn gnilnd scheduler[%d]: %d\n",
- i, rc);
+ if (*kgnilnd_tunables.kgn_thread_affinity) {
+ rc = kgnilnd_start_sd_threads();
+ if (rc != 0)
GOTO(failed, rc);
+ } else {
+ for (i = 0; i < *kgnilnd_tunables.kgn_sched_threads; i++) {
+ rc = kgnilnd_thread_start(kgnilnd_scheduler,
+ (void *)((long)i),
+ "kgnilnd_sd", i);
+ if (rc != 0) {
+ CERROR("Can't spawn gnilnd scheduler[%d]: %d\n",
+ i, rc);
+ GOTO(failed, rc);
+ }
}
}
}
}
-
-
/* flag everything initialised */
kgnilnd_data.kgn_init = GNILND_INIT_ALL;
/*****************************************************/
void
kgnilnd_base_shutdown(void)
{
- int i;
+ int i, j;
ENTRY;
while (CFS_FAIL_TIMEOUT(CFS_FAIL_GNI_PAUSE_SHUTDOWN, 1)) {};
for (i = 0; i < kgnilnd_data.kgn_ndevs; i++) {
kgn_device_t *dev = &kgnilnd_data.kgn_devices[i];
kgnilnd_cancel_wc_dgrams(dev);
+ kgnilnd_cancel_dgrams(dev);
kgnilnd_del_conn_or_peer(NULL, LNET_NID_ANY, GNILND_DEL_PEER, -ESHUTDOWN);
kgnilnd_wait_for_canceled_dgrams(dev);
}
+ /* We need to verify there are no conns left before we let the threads
+ * shut down otherwise we could clean up the peers but still have
+ * some outstanding conns due to orphaned datagram conns that are
+ * being cleaned up.
+ */
+ i = 2;
+ while (atomic_read(&kgnilnd_data.kgn_nconns) != 0) {
+ i++;
+
+ for(j = 0; j < kgnilnd_data.kgn_ndevs; ++j) {
+ kgn_device_t *dev = &kgnilnd_data.kgn_devices[j];
+ kgnilnd_schedule_device(dev);
+ }
+
+ CDEBUG(((i & (-i)) == i) ? D_WARNING : D_NET,
+ "Waiting for conns to be cleaned up %d\n",atomic_read(&kgnilnd_data.kgn_nconns));
+ set_current_state(TASK_UNINTERRUPTIBLE);
+ schedule_timeout(cfs_time_seconds(1));
+ }
/* Peer state all cleaned up BEFORE setting shutdown, so threads don't
* have to worry about shutdown races. NB connections may be created
* while there are still active connds, but these will be temporary
i++;
CDEBUG(((i & (-i)) == i) ? D_WARNING : D_NET,
"Waiting for ruhroh thread to terminate\n");
- cfs_pause(cfs_time_seconds(1));
+ set_current_state(TASK_UNINTERRUPTIBLE);
+ schedule_timeout(cfs_time_seconds(1));
}
/* Flag threads to terminate */
kgn_device_t *dev = &kgnilnd_data.kgn_devices[i];
/* should clear all the MDDs */
- kgnilnd_unmap_phys_fmablk(dev);
+ kgnilnd_unmap_fma_blocks(dev);
kgnilnd_schedule_device(dev);
wake_up_all(&dev->gnd_dgram_waitq);
wake_up_all(&kgnilnd_data.kgn_reaper_waitq);
spin_unlock(&kgnilnd_data.kgn_reaper_lock);
- kgnilnd_wakeup_rca_thread();
+ if (atomic_read(&kgnilnd_data.kgn_nthreads))
+ kgnilnd_wakeup_rca_thread();
/* Wait for threads to exit */
i = 2;
CDEBUG(((i & (-i)) == i) ? D_WARNING : D_NET, /* power of 2? */
"Waiting for %d threads to terminate\n",
atomic_read(&kgnilnd_data.kgn_nthreads));
- cfs_pause(cfs_time_seconds(1));
+ set_current_state(TASK_UNINTERRUPTIBLE);
+ schedule_timeout(cfs_time_seconds(1));
}
LASSERTF(atomic_read(&kgnilnd_data.kgn_npeers) == 0,
kgnilnd_free_phys_fmablk(dev);
}
- if (kgnilnd_data.kgn_mbox_cache != NULL) {
- i = cfs_mem_cache_destroy(kgnilnd_data.kgn_mbox_cache);
- LASSERTF(i == 0, "rc %d destroying kgn_mbox_cache\n", i);
- }
+ if (kgnilnd_data.kgn_mbox_cache != NULL)
+ kmem_cache_destroy(kgnilnd_data.kgn_mbox_cache);
- if (kgnilnd_data.kgn_rx_cache != NULL) {
- i = cfs_mem_cache_destroy(kgnilnd_data.kgn_rx_cache);
- LASSERTF(i == 0, "rc %d destroying kgn_rx_cache\n", i);
- }
+ if (kgnilnd_data.kgn_rx_cache != NULL)
+ kmem_cache_destroy(kgnilnd_data.kgn_rx_cache);
- if (kgnilnd_data.kgn_tx_cache != NULL) {
- i = cfs_mem_cache_destroy(kgnilnd_data.kgn_tx_cache);
- LASSERTF(i == 0, "rc %d destroying kgn_tx_cache\n", i);
- }
+ if (kgnilnd_data.kgn_tx_cache != NULL)
+ kmem_cache_destroy(kgnilnd_data.kgn_tx_cache);
- if (kgnilnd_data.kgn_tx_phys_cache != NULL) {
- i = cfs_mem_cache_destroy(kgnilnd_data.kgn_tx_phys_cache);
- LASSERTF(i == 0, "rc %d destroying kgn_tx_phys_cache\n", i);
- }
+ if (kgnilnd_data.kgn_tx_phys_cache != NULL)
+ kmem_cache_destroy(kgnilnd_data.kgn_tx_phys_cache);
- if (kgnilnd_data.kgn_dgram_cache != NULL) {
- i = cfs_mem_cache_destroy(kgnilnd_data.kgn_dgram_cache);
- LASSERTF(i == 0, "rc %d destroying kgn_dgram_cache\n", i);
- }
+ if (kgnilnd_data.kgn_dgram_cache != NULL)
+ kmem_cache_destroy(kgnilnd_data.kgn_dgram_cache);
if (kgnilnd_data.kgn_cksum_map_pages != NULL) {
for (i = 0; i < kgnilnd_data.kgn_cksum_npages; i++) {
atomic_read(&libcfs_kmemory));
kgnilnd_data.kgn_init = GNILND_INIT_NOTHING;
- PORTAL_MODULE_UNUSE;
+ module_put(THIS_MODULE);
EXIT;
}
}
/* Serialize with shutdown. */
- down(&kgnilnd_data.kgn_quiesce_sem);
+ mutex_lock(&kgnilnd_data.kgn_quiesce_mutex);
LIBCFS_ALLOC(net, sizeof(*net));
if (net == NULL) {
CERROR("could not allocate net for new interface instance\n");
- rc = -ENOMEM;
/* no need to cleanup the CDM... */
- GOTO(failed, rc);
+ GOTO(failed, rc = -ENOMEM);
}
INIT_LIST_HEAD(&net->gnn_list);
ni->ni_data = net;
timeout);
ni->ni_data = NULL;
LIBCFS_FREE(net, sizeof(*net));
- rc = -EINVAL;
- GOTO(failed, rc);
+ GOTO(failed, rc = -EINVAL);
} else
ni->ni_peertimeout = timeout;
/* we need a separate thread to call probe_wait_by_id until
* we get a function callback notifier from kgni */
- up(&kgnilnd_data.kgn_quiesce_sem);
+ mutex_unlock(&kgnilnd_data.kgn_quiesce_mutex);
RETURN(0);
failed:
- up(&kgnilnd_data.kgn_quiesce_sem);
+ mutex_unlock(&kgnilnd_data.kgn_quiesce_mutex);
kgnilnd_shutdown(ni);
RETURN(rc);
}
"init %d\n", kgnilnd_data.kgn_init);
/* Serialize with startup. */
- down(&kgnilnd_data.kgn_quiesce_sem);
+ mutex_lock(&kgnilnd_data.kgn_quiesce_mutex);
CDEBUG(D_MALLOC, "before NAL cleanup: kmem %d\n",
atomic_read(&libcfs_kmemory));
if (net == NULL) {
CERROR("got NULL net for ni %p\n", ni);
- rc = -EINVAL;
- GOTO(out, rc);
+ GOTO(out, rc = -EINVAL);
}
LASSERTF(ni == net->gnn_ni,
"Waiting for %d references to clear on net %d\n",
atomic_read(&net->gnn_refcount),
net->gnn_netnum);
- cfs_pause(cfs_time_seconds(1));
+ set_current_state(TASK_UNINTERRUPTIBLE);
+ schedule_timeout(cfs_time_seconds(1));
}
/* release ref from kgnilnd_startup */
CDEBUG(D_MALLOC, "after NAL cleanup: kmem %d\n",
atomic_read(&libcfs_kmemory));
- up(&kgnilnd_data.kgn_quiesce_sem);
+ mutex_unlock(&kgnilnd_data.kgn_quiesce_mutex);
EXIT;
- return;
}
-void __exit
-kgnilnd_module_fini(void)
+static void __exit kgnilnd_exit(void)
{
lnet_unregister_lnd(&the_kgnilnd);
kgnilnd_proc_fini();
kgnilnd_remove_sysctl();
- kgnilnd_tunables_fini();
}
-int __init
-kgnilnd_module_init(void)
+static int __init kgnilnd_init(void)
{
int rc;
}
MODULE_AUTHOR("Cray, Inc. <nic@cray.com>");
-MODULE_DESCRIPTION("Kernel Gemini LND v"KGNILND_BUILD_REV);
+MODULE_DESCRIPTION("Gemini LNet Network Driver");
+MODULE_VERSION(KGNILND_BUILD_REV);
MODULE_LICENSE("GPL");
-module_init(kgnilnd_module_init);
-module_exit(kgnilnd_module_fini);
+module_init(kgnilnd_init);
+module_exit(kgnilnd_exit);