* One WR for the LNet message
* And ibc_max_frags for the transfer WRs
*/
- unsigned int ret = 1 + conn->ibc_max_frags;
+ int ret;
+ int multiplier = 1 + conn->ibc_max_frags;
enum kib_dev_caps dev_caps = conn->ibc_hdev->ibh_dev->ibd_dev_caps;
/* FastReg needs two extra WRs for map and invalidate */
if (dev_caps & IBLND_DEV_CAPS_FASTREG_ENABLED)
- ret += 2;
+ multiplier += 2;
/* account for a maximum of ibc_queue_depth in-flight transfers */
- ret *= conn->ibc_queue_depth;
- return ret;
+ ret = multiplier * conn->ibc_queue_depth;
+
+ if (ret > conn->ibc_hdev->ibh_max_qp_wr) {
+ CDEBUG(D_NET, "peer_credits %u will result in send work "
+ "request size %d larger than maximum %d device "
+ "can handle\n", conn->ibc_queue_depth, ret,
+ conn->ibc_hdev->ibh_max_qp_wr);
+ conn->ibc_queue_depth =
+ conn->ibc_hdev->ibh_max_qp_wr / multiplier;
+ }
+
+ /* don't go beyond the maximum the device can handle */
+ return min(ret, conn->ibc_hdev->ibh_max_qp_wr);
}
struct kib_conn *
init_qp_attr->qp_type = IB_QPT_RC;
init_qp_attr->send_cq = cq;
init_qp_attr->recv_cq = cq;
+ /*
+ * kiblnd_send_wrs() can change the connection's queue depth if
+ * the maximum work requests for the device is maxed out
+ */
+ init_qp_attr->cap.max_send_wr = kiblnd_send_wrs(conn);
+ init_qp_attr->cap.max_recv_wr = IBLND_RECV_WRS(conn);
- conn->ibc_sched = sched;
-
- do {
- init_qp_attr->cap.max_send_wr = kiblnd_send_wrs(conn);
- init_qp_attr->cap.max_recv_wr = IBLND_RECV_WRS(conn);
-
- rc = rdma_create_qp(cmid, conn->ibc_hdev->ibh_pd, init_qp_attr);
- if (!rc || conn->ibc_queue_depth < 2)
- break;
-
- conn->ibc_queue_depth--;
- } while (rc);
-
+ rc = rdma_create_qp(cmid, conn->ibc_hdev->ibh_pd, init_qp_attr);
if (rc) {
CERROR("Can't create QP: %d, send_wr: %d, recv_wr: %d, "
"send_sge: %d, recv_sge: %d\n",
goto failed_2;
}
+ conn->ibc_sched = sched;
+
if (conn->ibc_queue_depth != peer_ni->ibp_queue_depth)
CWARN("peer %s - queue depth reduced from %u to %u"
" to allow for qp creation\n",
#endif
hdev->ibh_mr_size = dev_attr->max_mr_size;
+ hdev->ibh_max_qp_wr = dev_attr->max_qp_wr;
/* Setup device Memory Registration capabilities */
#ifdef HAVE_IB_DEVICE_OPS
return rc;
}
-static int kiblnd_dev_start_threads(struct kib_dev *dev, u32 *cpts, int ncpts)
+static int kiblnd_dev_start_threads(struct kib_dev *dev, bool newdev, u32 *cpts,
+ int ncpts)
{
int cpt;
int rc;
cpt = (cpts == NULL) ? i : cpts[i];
sched = kiblnd_data.kib_scheds[cpt];
- if (sched->ibs_nthreads > 0)
+ if (!newdev && sched->ibs_nthreads > 0)
continue;
rc = kiblnd_start_schedulers(kiblnd_data.kib_scheds[cpt]);
return 0;
}
+static struct kib_dev *
+kiblnd_dev_search(char *ifname)
+{
+ struct kib_dev *alias = NULL;
+ struct kib_dev *dev;
+ char *colon;
+ char *colon2;
+
+ colon = strchr(ifname, ':');
+ list_for_each_entry(dev, &kiblnd_data.kib_devs, ibd_list) {
+ if (strcmp(&dev->ibd_ifname[0], ifname) == 0)
+ return dev;
+
+ if (alias != NULL)
+ continue;
+
+ colon2 = strchr(dev->ibd_ifname, ':');
+ if (colon != NULL)
+ *colon = 0;
+ if (colon2 != NULL)
+ *colon2 = 0;
+
+ if (strcmp(&dev->ibd_ifname[0], ifname) == 0)
+ alias = dev;
+
+ if (colon != NULL)
+ *colon = ':';
+ if (colon2 != NULL)
+ *colon2 = ':';
+ }
+ return alias;
+}
+
static int
kiblnd_startup(struct lnet_ni *ni)
{
- char *ifname;
+ char *ifname = NULL;
struct lnet_inetdev *ifaces = NULL;
struct kib_dev *ibdev = NULL;
- struct kib_net *net;
- unsigned long flags;
- int rc;
+ struct kib_net *net = NULL;
+ unsigned long flags;
+ int rc;
int i;
+ bool newdev;
- LASSERT (ni->ni_net->net_lnd == &the_o2iblnd);
+ LASSERT(ni->ni_net->net_lnd == &the_o2iblnd);
- if (kiblnd_data.kib_init == IBLND_INIT_NOTHING) {
+ if (kiblnd_data.kib_init == IBLND_INIT_NOTHING) {
rc = kiblnd_base_startup(ni->ni_net_ns);
- if (rc != 0)
- return rc;
- }
+ if (rc != 0)
+ return rc;
+ }
- LIBCFS_ALLOC(net, sizeof(*net));
- ni->ni_data = net;
- if (net == NULL)
- goto failed;
+ LIBCFS_ALLOC(net, sizeof(*net));
+ ni->ni_data = net;
+ if (net == NULL) {
+ rc = -ENOMEM;
+ goto failed;
+ }
net->ibn_incarnation = ktime_get_real_ns() / NSEC_PER_USEC;
/* Use the IPoIB interface specified in 'networks=' */
if (ni->ni_interfaces[1] != NULL) {
CERROR("ko2iblnd: Multiple interfaces not supported\n");
+ rc = -EINVAL;
goto failed;
}
ifname = *kiblnd_tunables.kib_default_ipif;
}
- if (strlen(ifname) >= sizeof(ibdev->ibd_ifname)) {
- CERROR("IPoIB interface name too long: %s\n", ifname);
- goto failed;
- }
+ if (strlen(ifname) >= sizeof(ibdev->ibd_ifname)) {
+ CERROR("IPoIB interface name too long: %s\n", ifname);
+ rc = -E2BIG;
+ goto failed;
+ }
rc = lnet_inet_enumerate(&ifaces, ni->ni_net_ns);
if (rc < 0)
goto failed;
}
- LIBCFS_ALLOC(ibdev, sizeof(*ibdev));
- if (!ibdev) {
- rc = -ENOMEM;
- goto failed;
- }
+ ibdev = kiblnd_dev_search(ifname);
+ newdev = ibdev == NULL;
+ /* hmm...create kib_dev even for alias */
+ if (ibdev == NULL || strcmp(&ibdev->ibd_ifname[0], ifname) != 0) {
+ LIBCFS_ALLOC(ibdev, sizeof(*ibdev));
+ if (!ibdev) {
+ rc = -ENOMEM;
+ goto failed;
+ }
- ibdev->ibd_ifip = ifaces[i].li_ipaddr;
- strlcpy(ibdev->ibd_ifname, ifaces[i].li_name,
- sizeof(ibdev->ibd_ifname));
- ibdev->ibd_can_failover = !!(ifaces[i].li_flags & IFF_MASTER);
+ ibdev->ibd_ifip = ifaces[i].li_ipaddr;
+ strlcpy(ibdev->ibd_ifname, ifaces[i].li_name,
+ sizeof(ibdev->ibd_ifname));
+ ibdev->ibd_can_failover = !!(ifaces[i].li_flags & IFF_MASTER);
- INIT_LIST_HEAD(&ibdev->ibd_nets);
- INIT_LIST_HEAD(&ibdev->ibd_list); /* not yet in kib_devs */
- INIT_LIST_HEAD(&ibdev->ibd_fail_list);
+ INIT_LIST_HEAD(&ibdev->ibd_nets);
+ INIT_LIST_HEAD(&ibdev->ibd_list); /* not yet in kib_devs */
+ INIT_LIST_HEAD(&ibdev->ibd_fail_list);
- /* initialize the device */
- rc = kiblnd_dev_failover(ibdev, ni->ni_net_ns);
- if (rc) {
- CERROR("ko2iblnd: Can't initialize device: rc = %d\n", rc);
- goto failed;
- }
+ /* initialize the device */
+ rc = kiblnd_dev_failover(ibdev, ni->ni_net_ns);
+ if (rc) {
+ CERROR("ko2iblnd: Can't initialize device: rc = %d\n",
+ rc);
+ goto failed;
+ }
- list_add_tail(&ibdev->ibd_list, &kiblnd_data.kib_devs);
+ list_add_tail(&ibdev->ibd_list, &kiblnd_data.kib_devs);
+ }
net->ibn_dev = ibdev;
ni->ni_nid = LNET_MKNID(LNET_NIDNET(ni->ni_nid), ibdev->ibd_ifip);
ni->ni_dev_cpt = ifaces[i].li_cpt;
- rc = kiblnd_dev_start_threads(ibdev, ni->ni_cpts, ni->ni_ncpts);
+ rc = kiblnd_dev_start_threads(ibdev, newdev, ni->ni_cpts, ni->ni_ncpts);
if (rc != 0)
goto failed;
rc = kiblnd_net_init_pools(net, ni, ni->ni_cpts, ni->ni_ncpts);
- if (rc != 0) {
- CERROR("Failed to initialize NI pools: %d\n", rc);
- goto failed;
- }
+ if (rc != 0) {
+ CERROR("Failed to initialize NI pools: %d\n", rc);
+ goto failed;
+ }
write_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
ibdev->ibd_nnets++;
list_add_tail(&net->ibn_list, &ibdev->ibd_nets);
write_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
- net->ibn_init = IBLND_INIT_ALL;
+ net->ibn_init = IBLND_INIT_ALL;
- return 0;
+ return 0;
failed:
if (net != NULL && net->ibn_dev == NULL && ibdev != NULL)
- kiblnd_destroy_dev(ibdev);
+ kiblnd_destroy_dev(ibdev);
kfree(ifaces);
- kiblnd_shutdown(ni);
+ kiblnd_shutdown(ni);
- CDEBUG(D_NET, "kiblnd_startup failed\n");
- return -ENETDOWN;
+ CDEBUG(D_NET, "Configuration of device %s failed: rc = %d\n",
+ ifname ? ifname : "", rc);
+
+ return -ENETDOWN;
}
static struct lnet_lnd the_o2iblnd = {