lnet/klnds/viblnd/viblnd.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * Copyright (C) 2004 Cluster File Systems, Inc.
   5  *   Author: Eric Barton <eric@bartonsoftware.com>
   6  *   Author: Frank Zago <fzago@systemfabricworks.com>
   7  *
   8  *   This file is part of Lustre, http://www.lustre.org.
   9  *
  10  *   Lustre is free software; you can redistribute it and/or
  11  *   modify it under the terms of version 2 of the GNU General Public
  12  *   License as published by the Free Software Foundation.
  13  *
  14  *   Lustre is distributed in the hope that it will be useful,
  15  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  *   GNU General Public License for more details.
  18  *
  19  *   You should have received a copy of the GNU General Public License
  20  *   along with Lustre; if not, write to the Free Software
  21  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  22  *
  23  */
  24
  25 #include "vibnal.h"
  26
  27 nal_t                   kibnal_api;
  28 ptl_handle_ni_t         kibnal_ni;
  29 kib_data_t              kibnal_data;
  30 kib_tunables_t          kibnal_tunables;
  31
  32 #ifdef CONFIG_SYSCTL
  33 #define IBNAL_SYSCTL             202
  34
  35 #define IBNAL_SYSCTL_TIMEOUT     1
  36
  37 static ctl_table kibnal_ctl_table[] = {
  38         {IBNAL_SYSCTL_TIMEOUT, "timeout",
  39          &kibnal_tunables.kib_io_timeout, sizeof (int),
  40          0644, NULL, &proc_dointvec},
  41         { 0 }
  42 };
  43
  44 static ctl_table kibnal_top_ctl_table[] = {
  45         {IBNAL_SYSCTL, "vibnal", NULL, 0, 0555, kibnal_ctl_table},
  46         { 0 }
  47 };
  48 #endif
  49
  50 void vibnal_assert_wire_constants (void)
  51 {
  52         /* Wire protocol assertions generated by 'wirecheck'
  53          * running on Linux robert.bartonsoftware.com 2.6.5-1.358 #1 Sat May 8 09:04:50 EDT 2004 i686
  54          * with gcc version 3.3.3 20040412 (Red Hat Linux 3.3.3-7) */
  55
  56
  57         /* Constants... */
  58         CLASSERT (IBNAL_MSG_MAGIC == 0x0be91b91);
  59         CLASSERT (IBNAL_MSG_VERSION == 6);
  60         CLASSERT (IBNAL_MSG_CONNREQ == 0xc0);
  61         CLASSERT (IBNAL_MSG_CONNACK == 0xc1);
  62         CLASSERT (IBNAL_MSG_NOOP == 0xd0);
  63         CLASSERT (IBNAL_MSG_IMMEDIATE == 0xd1);
  64         CLASSERT (IBNAL_MSG_PUT_REQ == 0xd2);
  65         CLASSERT (IBNAL_MSG_PUT_NAK == 0xd3);
  66         CLASSERT (IBNAL_MSG_PUT_ACK == 0xd4);
  67         CLASSERT (IBNAL_MSG_PUT_DONE == 0xd5);
  68         CLASSERT (IBNAL_MSG_GET_REQ == 0xd6);
  69         CLASSERT (IBNAL_MSG_GET_DONE == 0xd7);
  70
  71         /* Checks for struct kib_connparams_t */
  72         CLASSERT ((int)sizeof(kib_connparams_t) == 12);
  73         CLASSERT ((int)offsetof(kib_connparams_t, ibcp_queue_depth) == 0);
  74         CLASSERT ((int)sizeof(((kib_connparams_t *)0)->ibcp_queue_depth) == 4);
  75         CLASSERT ((int)offsetof(kib_connparams_t, ibcp_max_msg_size) == 4);
  76         CLASSERT ((int)sizeof(((kib_connparams_t *)0)->ibcp_max_msg_size) == 4);
  77         CLASSERT ((int)offsetof(kib_connparams_t, ibcp_max_frags) == 8);
  78         CLASSERT ((int)sizeof(((kib_connparams_t *)0)->ibcp_max_frags) == 4);
  79
  80         /* Checks for struct kib_immediate_msg_t */
  81         CLASSERT ((int)sizeof(kib_immediate_msg_t) == 72);
  82         CLASSERT ((int)offsetof(kib_immediate_msg_t, ibim_hdr) == 0);
  83         CLASSERT ((int)sizeof(((kib_immediate_msg_t *)0)->ibim_hdr) == 72);
  84         CLASSERT ((int)offsetof(kib_immediate_msg_t, ibim_payload[13]) == 85);
  85         CLASSERT ((int)sizeof(((kib_immediate_msg_t *)0)->ibim_payload[13]) == 1);
  86
  87         /* Checks for struct kib_rdma_frag_t */
  88         CLASSERT ((int)sizeof(kib_rdma_frag_t) == 12);
  89         CLASSERT ((int)offsetof(kib_rdma_frag_t, rf_nob) == 0);
  90         CLASSERT ((int)sizeof(((kib_rdma_frag_t *)0)->rf_nob) == 4);
  91         CLASSERT ((int)offsetof(kib_rdma_frag_t, rf_addr_lo) == 4);
  92         CLASSERT ((int)sizeof(((kib_rdma_frag_t *)0)->rf_addr_lo) == 4);
  93         CLASSERT ((int)offsetof(kib_rdma_frag_t, rf_addr_hi) == 8);
  94         CLASSERT ((int)sizeof(((kib_rdma_frag_t *)0)->rf_addr_hi) == 4);
  95
  96         /* Checks for struct kib_rdma_desc_t */
  97         CLASSERT ((int)sizeof(kib_rdma_desc_t) == 8);
  98         CLASSERT ((int)offsetof(kib_rdma_desc_t, rd_key) == 0);
  99         CLASSERT ((int)sizeof(((kib_rdma_desc_t *)0)->rd_key) == 4);
 100         CLASSERT ((int)offsetof(kib_rdma_desc_t, rd_nfrag) == 4);
 101         CLASSERT ((int)sizeof(((kib_rdma_desc_t *)0)->rd_nfrag) == 4);
 102         CLASSERT ((int)offsetof(kib_rdma_desc_t, rd_frags[13]) == 164);
 103         CLASSERT ((int)sizeof(((kib_rdma_desc_t *)0)->rd_frags[13]) == 12);
 104
 105         /* Checks for struct kib_putreq_msg_t */
 106         CLASSERT ((int)sizeof(kib_putreq_msg_t) == 80);
 107         CLASSERT ((int)offsetof(kib_putreq_msg_t, ibprm_hdr) == 0);
 108         CLASSERT ((int)sizeof(((kib_putreq_msg_t *)0)->ibprm_hdr) == 72);
 109         CLASSERT ((int)offsetof(kib_putreq_msg_t, ibprm_cookie) == 72);
 110         CLASSERT ((int)sizeof(((kib_putreq_msg_t *)0)->ibprm_cookie) == 8);
 111
 112         /* Checks for struct kib_putack_msg_t */
 113         CLASSERT ((int)sizeof(kib_putack_msg_t) == 24);
 114         CLASSERT ((int)offsetof(kib_putack_msg_t, ibpam_src_cookie) == 0);
 115         CLASSERT ((int)sizeof(((kib_putack_msg_t *)0)->ibpam_src_cookie) == 8);
 116         CLASSERT ((int)offsetof(kib_putack_msg_t, ibpam_dst_cookie) == 8);
 117         CLASSERT ((int)sizeof(((kib_putack_msg_t *)0)->ibpam_dst_cookie) == 8);
 118         CLASSERT ((int)offsetof(kib_putack_msg_t, ibpam_rd) == 16);
 119         CLASSERT ((int)sizeof(((kib_putack_msg_t *)0)->ibpam_rd) == 8);
 120
 121         /* Checks for struct kib_get_msg_t */
 122         CLASSERT ((int)sizeof(kib_get_msg_t) == 88);
 123         CLASSERT ((int)offsetof(kib_get_msg_t, ibgm_hdr) == 0);
 124         CLASSERT ((int)sizeof(((kib_get_msg_t *)0)->ibgm_hdr) == 72);
 125         CLASSERT ((int)offsetof(kib_get_msg_t, ibgm_cookie) == 72);
 126         CLASSERT ((int)sizeof(((kib_get_msg_t *)0)->ibgm_cookie) == 8);
 127         CLASSERT ((int)offsetof(kib_get_msg_t, ibgm_rd) == 80);
 128         CLASSERT ((int)sizeof(((kib_get_msg_t *)0)->ibgm_rd) == 8);
 129
 130         /* Checks for struct kib_completion_msg_t */
 131         CLASSERT ((int)sizeof(kib_completion_msg_t) == 12);
 132         CLASSERT ((int)offsetof(kib_completion_msg_t, ibcm_cookie) == 0);
 133         CLASSERT ((int)sizeof(((kib_completion_msg_t *)0)->ibcm_cookie) == 8);
 134         CLASSERT ((int)offsetof(kib_completion_msg_t, ibcm_status) == 8);
 135         CLASSERT ((int)sizeof(((kib_completion_msg_t *)0)->ibcm_status) == 4);
 136
 137         /* Checks for struct kib_msg_t */
 138         CLASSERT ((int)sizeof(kib_msg_t) == 144);
 139         CLASSERT ((int)offsetof(kib_msg_t, ibm_magic) == 0);
 140         CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_magic) == 4);
 141         CLASSERT ((int)offsetof(kib_msg_t, ibm_version) == 4);
 142         CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_version) == 2);
 143         CLASSERT ((int)offsetof(kib_msg_t, ibm_type) == 6);
 144         CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_type) == 1);
 145         CLASSERT ((int)offsetof(kib_msg_t, ibm_credits) == 7);
 146         CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_credits) == 1);
 147         CLASSERT ((int)offsetof(kib_msg_t, ibm_nob) == 8);
 148         CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_nob) == 4);
 149         CLASSERT ((int)offsetof(kib_msg_t, ibm_cksum) == 12);
 150         CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_cksum) == 4);
 151         CLASSERT ((int)offsetof(kib_msg_t, ibm_srcnid) == 16);
 152         CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_srcnid) == 8);
 153         CLASSERT ((int)offsetof(kib_msg_t, ibm_srcstamp) == 24);
 154         CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_srcstamp) == 8);
 155         CLASSERT ((int)offsetof(kib_msg_t, ibm_dstnid) == 32);
 156         CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_dstnid) == 8);
 157         CLASSERT ((int)offsetof(kib_msg_t, ibm_dststamp) == 40);
 158         CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_dststamp) == 8);
 159         CLASSERT ((int)offsetof(kib_msg_t, ibm_seq) == 48);
 160         CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_seq) == 8);
 161         CLASSERT ((int)offsetof(kib_msg_t, ibm_u.connparams) == 56);
 162         CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_u.connparams) == 12);
 163         CLASSERT ((int)offsetof(kib_msg_t, ibm_u.immediate) == 56);
 164         CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_u.immediate) == 72);
 165         CLASSERT ((int)offsetof(kib_msg_t, ibm_u.putreq) == 56);
 166         CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_u.putreq) == 80);
 167         CLASSERT ((int)offsetof(kib_msg_t, ibm_u.putack) == 56);
 168         CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_u.putack) == 24);
 169         CLASSERT ((int)offsetof(kib_msg_t, ibm_u.get) == 56);
 170         CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_u.get) == 88);
 171         CLASSERT ((int)offsetof(kib_msg_t, ibm_u.completion) == 56);
 172         CLASSERT ((int)sizeof(((kib_msg_t *)0)->ibm_u.completion) == 12);
 173 }
 174
 175 void
 176 kibnal_pause(int ticks)
 177 {
 178         set_current_state(TASK_UNINTERRUPTIBLE);
 179         schedule_timeout(ticks);
 180 }
 181
 182 __u32
 183 kibnal_cksum (void *ptr, int nob)
 184 {
 185         char  *c  = ptr;
 186         __u32  sum = 0;
 187
 188         while (nob-- > 0)
 189                 sum = ((sum << 1) | (sum >> 31)) + *c++;
 190
 191         /* ensure I don't return 0 (== no checksum) */
 192         return (sum == 0) ? 1 : sum;
 193 }
 194
 195 void
 196 kibnal_init_msg(kib_msg_t *msg, int type, int body_nob)
 197 {
 198         msg->ibm_type = type;
 199         msg->ibm_nob  = offsetof(kib_msg_t, ibm_u) + body_nob;
 200 }
 201
 202 void
 203 kibnal_pack_msg(kib_msg_t *msg, int credits, ptl_nid_t dstnid,
 204                 __u64 dststamp, __u64 seq)
 205 {
 206         /* CAVEAT EMPTOR! all message fields not set here should have been
 207          * initialised previously. */
 208         msg->ibm_magic    = IBNAL_MSG_MAGIC;
 209         msg->ibm_version  = IBNAL_MSG_VERSION;
 210         /*   ibm_type */
 211         msg->ibm_credits  = credits;
 212         /*   ibm_nob */
 213         msg->ibm_cksum    = 0;
 214         msg->ibm_srcnid   = kibnal_lib.libnal_ni.ni_pid.nid;
 215         msg->ibm_srcstamp = kibnal_data.kib_incarnation;
 216         msg->ibm_dstnid   = dstnid;
 217         msg->ibm_dststamp = dststamp;
 218         msg->ibm_seq      = seq;
 219 #if IBNAL_CKSUM
 220         /* NB ibm_cksum zero while computing cksum */
 221         msg->ibm_cksum    = kibnal_cksum(msg, msg->ibm_nob);
 222 #endif
 223 }
 224
 225 int
 226 kibnal_unpack_msg(kib_msg_t *msg, int nob)
 227 {
 228         const int hdr_size = offsetof(kib_msg_t, ibm_u);
 229         __u32     msg_cksum;
 230         int       flip;
 231         int       msg_nob;
 232         int       i;
 233         int       n;
 234
 235         /* 6 bytes are enough to have received magic + version */
 236         if (nob < 6) {
 237                 CERROR("Short message: %d\n", nob);
 238                 return -EPROTO;
 239         }
 240
 241         if (msg->ibm_magic == IBNAL_MSG_MAGIC) {
 242                 flip = 0;
 243         } else if (msg->ibm_magic == __swab32(IBNAL_MSG_MAGIC)) {
 244                 flip = 1;
 245         } else {
 246                 CERROR("Bad magic: %08x\n", msg->ibm_magic);
 247                 return -EPROTO;
 248         }
 249
 250         if (msg->ibm_version !=
 251             (flip ? __swab16(IBNAL_MSG_VERSION) : IBNAL_MSG_VERSION)) {
 252                 CERROR("Bad version: %d\n", msg->ibm_version);
 253                 return -EPROTO;
 254         }
 255
 256         if (nob < hdr_size) {
 257                 CERROR("Short message: %d\n", nob);
 258                 return -EPROTO;
 259         }
 260
 261         msg_nob = flip ? __swab32(msg->ibm_nob) : msg->ibm_nob;
 262         if (msg_nob > nob) {
 263                 CERROR("Short message: got %d, wanted %d\n", nob, msg_nob);
 264                 return -EPROTO;
 265         }
 266
 267         /* checksum must be computed with ibm_cksum zero and BEFORE anything
 268          * gets flipped */
 269         msg_cksum = flip ? __swab32(msg->ibm_cksum) : msg->ibm_cksum;
 270         msg->ibm_cksum = 0;
 271         if (msg_cksum != 0 &&
 272             msg_cksum != kibnal_cksum(msg, msg_nob)) {
 273                 CERROR("Bad checksum\n");
 274                 return -EPROTO;
 275         }
 276         msg->ibm_cksum = msg_cksum;
 277
 278         if (flip) {
 279                 /* leave magic unflipped as a clue to peer endianness */
 280                 __swab16s(&msg->ibm_version);
 281                 CLASSERT (sizeof(msg->ibm_type) == 1);
 282                 CLASSERT (sizeof(msg->ibm_credits) == 1);
 283                 msg->ibm_nob = msg_nob;
 284                 __swab64s(&msg->ibm_srcnid);
 285                 __swab64s(&msg->ibm_srcstamp);
 286                 __swab64s(&msg->ibm_dstnid);
 287                 __swab64s(&msg->ibm_dststamp);
 288                 __swab64s(&msg->ibm_seq);
 289         }
 290
 291         if (msg->ibm_srcnid == PTL_NID_ANY) {
 292                 CERROR("Bad src nid: "LPX64"\n", msg->ibm_srcnid);
 293                 return -EPROTO;
 294         }
 295
 296         switch (msg->ibm_type) {
 297         default:
 298                 CERROR("Unknown message type %x\n", msg->ibm_type);
 299                 return -EPROTO;
 300
 301         case IBNAL_MSG_NOOP:
 302                 break;
 303
 304         case IBNAL_MSG_IMMEDIATE:
 305                 if (msg_nob < offsetof(kib_msg_t, ibm_u.immediate.ibim_payload[0])) {
 306                         CERROR("Short IMMEDIATE: %d(%d)\n", msg_nob,
 307                                (int)offsetof(kib_msg_t, ibm_u.immediate.ibim_payload[0]));
 308                         return -EPROTO;
 309                 }
 310                 break;
 311
 312         case IBNAL_MSG_PUT_REQ:
 313                 if (msg_nob < sizeof(msg->ibm_u.putreq)) {
 314                         CERROR("Short PUT_REQ: %d(%d)\n", msg_nob,
 315                                (int)(hdr_size + sizeof(msg->ibm_u.putreq)));
 316                         return -EPROTO;
 317                 }
 318                 break;
 319
 320         case IBNAL_MSG_PUT_ACK:
 321                 if (msg_nob < offsetof(kib_msg_t, ibm_u.putack.ibpam_rd.rd_frags[0])) {
 322                         CERROR("Short PUT_ACK: %d(%d)\n", msg_nob,
 323                                (int)offsetof(kib_msg_t, ibm_u.putack.ibpam_rd.rd_frags[0]));
 324                         return -EPROTO;
 325                 }
 326
 327                 if (flip) {
 328                         __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_key);
 329                         __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_nfrag);
 330                 }
 331
 332                 n = msg->ibm_u.putack.ibpam_rd.rd_nfrag;
 333                 if (n <= 0 || n > IBNAL_MAX_RDMA_FRAGS) {
 334                         CERROR("Bad PUT_ACK nfrags: %d, should be 0 < n <= %d\n",
 335                                n, IBNAL_MAX_RDMA_FRAGS);
 336                         return -EPROTO;
 337                 }
 338
 339                 if (msg_nob < offsetof(kib_msg_t, ibm_u.putack.ibpam_rd.rd_frags[n])) {
 340                         CERROR("Short PUT_ACK: %d(%d)\n", msg_nob,
 341                                (int)offsetof(kib_msg_t, ibm_u.putack.ibpam_rd.rd_frags[n]));
 342                         return -EPROTO;
 343                 }
 344
 345                 if (flip)
 346                         for (i = 0; i < n; i++) {
 347                                 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_frags[i].rf_nob);
 348                                 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_frags[i].rf_addr_lo);
 349                                 __swab32s(&msg->ibm_u.putack.ibpam_rd.rd_frags[i].rf_addr_hi);
 350                         }
 351                 break;
 352
 353         case IBNAL_MSG_GET_REQ:
 354                 if (msg_nob < hdr_size + sizeof(msg->ibm_u.get)) {
 355                         CERROR("Short GET_REQ: %d(%d)\n", msg_nob,
 356                                (int)(hdr_size + sizeof(msg->ibm_u.get)));
 357                         return -EPROTO;
 358                 }
 359                 if (flip) {
 360                         __swab32s(&msg->ibm_u.get.ibgm_rd.rd_key);
 361                         __swab32s(&msg->ibm_u.get.ibgm_rd.rd_nfrag);
 362                 }
 363
 364                 n = msg->ibm_u.get.ibgm_rd.rd_nfrag;
 365                 if (n <= 0 || n > IBNAL_MAX_RDMA_FRAGS) {
 366                         CERROR("Bad GET_REQ nfrags: %d, should be 0 < n <= %d\n",
 367                                n, IBNAL_MAX_RDMA_FRAGS);
 368                         return -EPROTO;
 369                 }
 370
 371                 if (msg_nob < offsetof(kib_msg_t, ibm_u.get.ibgm_rd.rd_frags[n])) {
 372                         CERROR("Short GET_REQ: %d(%d)\n", msg_nob,
 373                                (int)offsetof(kib_msg_t, ibm_u.get.ibgm_rd.rd_frags[n]));
 374                         return -EPROTO;
 375                 }
 376
 377                 if (flip)
 378                         for (i = 0; i < msg->ibm_u.get.ibgm_rd.rd_nfrag; i++) {
 379                                 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_frags[i].rf_nob);
 380                                 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_frags[i].rf_addr_lo);
 381                                 __swab32s(&msg->ibm_u.get.ibgm_rd.rd_frags[i].rf_addr_hi);
 382                         }
 383                 break;
 384
 385         case IBNAL_MSG_PUT_NAK:
 386         case IBNAL_MSG_PUT_DONE:
 387         case IBNAL_MSG_GET_DONE:
 388                 if (msg_nob < hdr_size + sizeof(msg->ibm_u.completion)) {
 389                         CERROR("Short RDMA completion: %d(%d)\n", msg_nob,
 390                                (int)(hdr_size + sizeof(msg->ibm_u.completion)));
 391                         return -EPROTO;
 392                 }
 393                 if (flip)
 394                         __swab32s(&msg->ibm_u.completion.ibcm_status);
 395                 break;
 396
 397         case IBNAL_MSG_CONNREQ:
 398         case IBNAL_MSG_CONNACK:
 399                 if (msg_nob < hdr_size + sizeof(msg->ibm_u.connparams)) {
 400                         CERROR("Short connreq/ack: %d(%d)\n", msg_nob,
 401                                (int)(hdr_size + sizeof(msg->ibm_u.connparams)));
 402                         return -EPROTO;
 403                 }
 404                 if (flip) {
 405                         __swab32s(&msg->ibm_u.connparams.ibcp_queue_depth);
 406                         __swab32s(&msg->ibm_u.connparams.ibcp_max_msg_size);
 407                         __swab32s(&msg->ibm_u.connparams.ibcp_max_frags);
 408                 }
 409                 break;
 410         }
 411         return 0;
 412 }
 413
 414 int
 415 kibnal_set_mynid(ptl_nid_t nid)
 416 {
 417         static cm_listen_data_t info;           /* protected by kib_nid_mutex */
 418
 419         lib_ni_t        *ni = &kibnal_lib.libnal_ni;
 420         int              rc;
 421         cm_return_t      cmrc;
 422
 423         CDEBUG(D_IOCTL, "setting mynid to "LPX64" (old nid="LPX64")\n",
 424                nid, ni->ni_pid.nid);
 425
 426         down (&kibnal_data.kib_nid_mutex);
 427
 428         if (nid == ni->ni_pid.nid) {
 429                 /* no change of NID */
 430                 up (&kibnal_data.kib_nid_mutex);
 431                 return (0);
 432         }
 433
 434         CDEBUG(D_NET, "NID "LPX64"("LPX64")\n", ni->ni_pid.nid, nid);
 435
 436         if (kibnal_data.kib_listen_handle != NULL) {
 437                 cmrc = cm_cancel(kibnal_data.kib_listen_handle);
 438                 if (cmrc != cm_stat_success)
 439                         CERROR ("Error %d stopping listener\n", cmrc);
 440
 441                 kibnal_pause(HZ/10);            /* ensure no more callbacks */
 442
 443                 cmrc = cm_destroy_cep(kibnal_data.kib_listen_handle);
 444                 if (cmrc != vv_return_ok)
 445                         CERROR ("Error %d destroying CEP\n", cmrc);
 446
 447                 kibnal_data.kib_listen_handle = NULL;
 448         }
 449
 450         /* Change NID.  NB queued passive connection requests (if any) will be
 451          * rejected with an incorrect destination NID */
 452         ni->ni_pid.nid = nid;
 453         kibnal_data.kib_incarnation++;
 454         mb();
 455
 456         /* Delete all existing peers and their connections after new
 457          * NID/incarnation set to ensure no old connections in our brave
 458          * new world. */
 459         kibnal_del_peer (PTL_NID_ANY, 0);
 460
 461         if (ni->ni_pid.nid != PTL_NID_ANY) {    /* got a new NID to install */
 462                 kibnal_data.kib_listen_handle =
 463                         cm_create_cep(cm_cep_transp_rc);
 464                 if (kibnal_data.kib_listen_handle == NULL) {
 465                         CERROR ("Can't create listen CEP\n");
 466                         rc = -ENOMEM;
 467                         goto failed_0;
 468                 }
 469
 470                 CDEBUG(D_NET, "Created CEP %p for listening\n",
 471                        kibnal_data.kib_listen_handle);
 472
 473                 memset(&info, 0, sizeof(info));
 474                 info.listen_addr.end_pt.sid = kibnal_data.kib_svc_id;
 475
 476                 cmrc = cm_listen(kibnal_data.kib_listen_handle, &info,
 477                                  kibnal_listen_callback, NULL);
 478                 if (cmrc != 0) {
 479                         CERROR ("cm_listen error: %d\n", cmrc);
 480                         rc = -EINVAL;
 481                         goto failed_1;
 482                 }
 483         }
 484
 485         up (&kibnal_data.kib_nid_mutex);
 486         return (0);
 487
 488  failed_1:
 489         cmrc = cm_destroy_cep(kibnal_data.kib_listen_handle);
 490         LASSERT (cmrc == cm_stat_success);
 491         kibnal_data.kib_listen_handle = NULL;
 492  failed_0:
 493         ni->ni_pid.nid = PTL_NID_ANY;
 494         kibnal_data.kib_incarnation++;
 495         mb();
 496         kibnal_del_peer (PTL_NID_ANY, 0);
 497         up (&kibnal_data.kib_nid_mutex);
 498         return rc;
 499 }
 500
 501 kib_peer_t *
 502 kibnal_create_peer (ptl_nid_t nid)
 503 {
 504         kib_peer_t *peer;
 505
 506         LASSERT (nid != PTL_NID_ANY);
 507
 508         PORTAL_ALLOC(peer, sizeof (*peer));
 509         if (peer == NULL) {
 510                 CERROR("Canot allocate perr\n");
 511                 return (NULL);
 512         }
 513
 514         memset(peer, 0, sizeof(*peer));         /* zero flags etc */
 515
 516         peer->ibp_nid = nid;
 517         atomic_set (&peer->ibp_refcount, 1);    /* 1 ref for caller */
 518
 519         INIT_LIST_HEAD (&peer->ibp_list);       /* not in the peer table yet */
 520         INIT_LIST_HEAD (&peer->ibp_conns);
 521         INIT_LIST_HEAD (&peer->ibp_tx_queue);
 522
 523         peer->ibp_reconnect_time = jiffies;
 524         peer->ibp_reconnect_interval = IBNAL_MIN_RECONNECT_INTERVAL;
 525
 526         atomic_inc (&kibnal_data.kib_npeers);
 527         if (atomic_read(&kibnal_data.kib_npeers) <= IBNAL_CONCURRENT_PEERS)
 528                 return peer;
 529
 530         CERROR("Too many peers: CQ will overflow\n");
 531         kibnal_peer_decref(peer);
 532         return NULL;
 533 }
 534
 535 void
 536 kibnal_destroy_peer (kib_peer_t *peer)
 537 {
 538
 539         LASSERT (atomic_read (&peer->ibp_refcount) == 0);
 540         LASSERT (peer->ibp_persistence == 0);
 541         LASSERT (!kibnal_peer_active(peer));
 542         LASSERT (peer->ibp_connecting == 0);
 543         LASSERT (list_empty (&peer->ibp_conns));
 544         LASSERT (list_empty (&peer->ibp_tx_queue));
 545
 546         PORTAL_FREE (peer, sizeof (*peer));
 547
 548         /* NB a peer's connections keep a reference on their peer until
 549          * they are destroyed, so we can be assured that _all_ state to do
 550          * with this peer has been cleaned up when its refcount drops to
 551          * zero. */
 552         atomic_dec (&kibnal_data.kib_npeers);
 553 }
 554
 555 /* the caller is responsible for accounting for the additional reference
 556  * that this creates */
 557 kib_peer_t *
 558 kibnal_find_peer_locked (ptl_nid_t nid)
 559 {
 560         struct list_head *peer_list = kibnal_nid2peerlist (nid);
 561         struct list_head *tmp;
 562         kib_peer_t       *peer;
 563
 564         list_for_each (tmp, peer_list) {
 565
 566                 peer = list_entry (tmp, kib_peer_t, ibp_list);
 567
 568                 LASSERT (peer->ibp_persistence != 0 || /* persistent peer */
 569                          peer->ibp_connecting != 0 || /* creating conns */
 570                          !list_empty (&peer->ibp_conns));  /* active conn */
 571
 572                 if (peer->ibp_nid != nid)
 573                         continue;
 574
 575                 CDEBUG(D_NET, "got peer [%p] -> "LPX64" (%d)\n",
 576                        peer, nid, atomic_read (&peer->ibp_refcount));
 577                 return (peer);
 578         }
 579         return (NULL);
 580 }
 581
 582 void
 583 kibnal_unlink_peer_locked (kib_peer_t *peer)
 584 {
 585         LASSERT (peer->ibp_persistence == 0);
 586         LASSERT (list_empty(&peer->ibp_conns));
 587
 588         LASSERT (kibnal_peer_active(peer));
 589         list_del_init (&peer->ibp_list);
 590         /* lose peerlist's ref */
 591         kibnal_peer_decref(peer);
 592 }
 593
 594 int
 595 kibnal_get_peer_info (int index, ptl_nid_t *nidp, __u32 *ipp,
 596                       int *persistencep)
 597 {
 598         kib_peer_t        *peer;
 599         struct list_head  *ptmp;
 600         int                i;
 601         unsigned long      flags;
 602
 603         read_lock_irqsave(&kibnal_data.kib_global_lock, flags);
 604
 605         for (i = 0; i < kibnal_data.kib_peer_hash_size; i++) {
 606
 607                 list_for_each (ptmp, &kibnal_data.kib_peers[i]) {
 608
 609                         peer = list_entry (ptmp, kib_peer_t, ibp_list);
 610                         LASSERT (peer->ibp_persistence != 0 ||
 611                                  peer->ibp_connecting != 0 ||
 612                                  !list_empty (&peer->ibp_conns));
 613
 614                         if (index-- > 0)
 615                                 continue;
 616
 617                         *nidp = peer->ibp_nid;
 618                         *ipp = peer->ibp_ip;
 619                         *persistencep = peer->ibp_persistence;
 620
 621                         read_unlock_irqrestore(&kibnal_data.kib_global_lock,
 622                                                flags);
 623                         return (0);
 624                 }
 625         }
 626
 627         read_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
 628         return (-ENOENT);
 629 }
 630
 631 int
 632 kibnal_add_persistent_peer (ptl_nid_t nid, __u32 ip)
 633 {
 634         kib_peer_t        *peer;
 635         kib_peer_t        *peer2;
 636         unsigned long      flags;
 637
 638         CDEBUG(D_NET, LPX64"@%08x\n", nid, ip);
 639
 640         if (nid == PTL_NID_ANY)
 641                 return (-EINVAL);
 642
 643         peer = kibnal_create_peer (nid);
 644         if (peer == NULL)
 645                 return (-ENOMEM);
 646
 647         write_lock_irqsave(&kibnal_data.kib_global_lock, flags);
 648
 649         peer2 = kibnal_find_peer_locked (nid);
 650         if (peer2 != NULL) {
 651                 kibnal_peer_decref (peer);
 652                 peer = peer2;
 653         } else {
 654                 /* peer table takes existing ref on peer */
 655                 list_add_tail (&peer->ibp_list,
 656                                kibnal_nid2peerlist (nid));
 657         }
 658
 659         peer->ibp_ip = ip;
 660         peer->ibp_persistence++;
 661
 662         write_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
 663         return (0);
 664 }
 665
 666 void
 667 kibnal_del_peer_locked (kib_peer_t *peer, int single_share)
 668 {
 669         struct list_head *ctmp;
 670         struct list_head *cnxt;
 671         kib_conn_t       *conn;
 672
 673         if (!single_share)
 674                 peer->ibp_persistence = 0;
 675         else if (peer->ibp_persistence > 0)
 676                 peer->ibp_persistence--;
 677
 678         if (peer->ibp_persistence != 0)
 679                 return;
 680
 681         if (list_empty(&peer->ibp_conns)) {
 682                 kibnal_unlink_peer_locked(peer);
 683         } else {
 684                 list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
 685                         conn = list_entry(ctmp, kib_conn_t, ibc_list);
 686
 687                         kibnal_close_conn_locked (conn, 0);
 688                 }
 689                 /* NB peer is no longer persistent; closing its last conn
 690                  * unlinked it. */
 691         }
 692         /* NB peer now unlinked; might even be freed if the peer table had the
 693          * last ref on it. */
 694 }
 695
 696 int
 697 kibnal_del_peer (ptl_nid_t nid, int single_share)
 698 {
 699         struct list_head  *ptmp;
 700         struct list_head  *pnxt;
 701         kib_peer_t        *peer;
 702         int                lo;
 703         int                hi;
 704         int                i;
 705         unsigned long      flags;
 706         int                rc = -ENOENT;
 707
 708         write_lock_irqsave(&kibnal_data.kib_global_lock, flags);
 709
 710         if (nid != PTL_NID_ANY)
 711                 lo = hi = kibnal_nid2peerlist(nid) - kibnal_data.kib_peers;
 712         else {
 713                 lo = 0;
 714                 hi = kibnal_data.kib_peer_hash_size - 1;
 715         }
 716
 717         for (i = lo; i <= hi; i++) {
 718                 list_for_each_safe (ptmp, pnxt, &kibnal_data.kib_peers[i]) {
 719                         peer = list_entry (ptmp, kib_peer_t, ibp_list);
 720                         LASSERT (peer->ibp_persistence != 0 ||
 721                                  peer->ibp_connecting != 0 ||
 722                                  !list_empty (&peer->ibp_conns));
 723
 724                         if (!(nid == PTL_NID_ANY || peer->ibp_nid == nid))
 725                                 continue;
 726
 727                         kibnal_del_peer_locked (peer, single_share);
 728                         rc = 0;         /* matched something */
 729
 730                         if (single_share)
 731                                 goto out;
 732                 }
 733         }
 734  out:
 735         write_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
 736         return (rc);
 737 }
 738
 739 kib_conn_t *
 740 kibnal_get_conn_by_idx (int index)
 741 {
 742         kib_peer_t        *peer;
 743         struct list_head  *ptmp;
 744         kib_conn_t        *conn;
 745         struct list_head  *ctmp;
 746         int                i;
 747         unsigned long      flags;
 748
 749         read_lock_irqsave(&kibnal_data.kib_global_lock, flags);
 750
 751         for (i = 0; i < kibnal_data.kib_peer_hash_size; i++) {
 752                 list_for_each (ptmp, &kibnal_data.kib_peers[i]) {
 753
 754                         peer = list_entry (ptmp, kib_peer_t, ibp_list);
 755                         LASSERT (peer->ibp_persistence > 0 ||
 756                                  peer->ibp_connecting != 0 ||
 757                                  !list_empty (&peer->ibp_conns));
 758
 759                         list_for_each (ctmp, &peer->ibp_conns) {
 760                                 if (index-- > 0)
 761                                         continue;
 762
 763                                 conn = list_entry (ctmp, kib_conn_t, ibc_list);
 764                                 kibnal_conn_addref(conn);
 765                                 read_unlock_irqrestore(&kibnal_data.kib_global_lock,
 766                                                        flags);
 767                                 return (conn);
 768                         }
 769                 }
 770         }
 771
 772         read_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
 773         return (NULL);
 774 }
 775
 776 int
 777 kibnal_set_qp_state (kib_conn_t *conn, vv_qp_state_t new_state)
 778 {
 779         static vv_qp_attr_t attr;
 780
 781         kib_connvars_t   *cv = conn->ibc_connvars;
 782         vv_return_t       vvrc;
 783
 784         /* Only called by connd => static OK */
 785         LASSERT (!in_interrupt());
 786         LASSERT (current == kibnal_data.kib_connd);
 787
 788         memset(&attr, 0, sizeof(attr));
 789
 790         switch (new_state) {
 791         default:
 792                 LBUG();
 793
 794         case vv_qp_state_init: {
 795                 struct vv_qp_modify_init_st *init = &attr.modify.params.init;
 796
 797                 init->p_key_indx     = cv->cv_pkey_index;
 798                 init->phy_port_num   = cv->cv_port;
 799                 init->q_key          = IBNAL_QKEY; /* XXX but VV_QP_AT_Q_KEY not set! */
 800                 init->access_control = vv_acc_r_mem_read |
 801                                        vv_acc_r_mem_write; /* XXX vv_acc_l_mem_write ? */
 802
 803                 attr.modify.vv_qp_attr_mask = VV_QP_AT_P_KEY_IX |
 804                                               VV_QP_AT_PHY_PORT_NUM |
 805                                               VV_QP_AT_ACCESS_CON_F;
 806                 break;
 807         }
 808         case vv_qp_state_rtr: {
 809                 struct vv_qp_modify_rtr_st *rtr = &attr.modify.params.rtr;
 810                 vv_add_vec_t               *av  = &rtr->remote_add_vec;
 811
 812                 av->dlid                      = cv->cv_path.dlid;
 813                 av->grh_flag                  = (!IBNAL_LOCAL_SUB);
 814                 av->max_static_rate           = IBNAL_R_2_STATIC_RATE(cv->cv_path.rate);
 815                 av->service_level             = cv->cv_path.sl;
 816                 av->source_path_bit           = IBNAL_SOURCE_PATH_BIT;
 817                 av->pmtu                      = cv->cv_path.mtu;
 818                 av->rnr_retry_count           = cv->cv_rnr_count;
 819                 av->global_dest.traffic_class = cv->cv_path.traffic_class;
 820                 av->global_dest.hope_limit    = cv->cv_path.hop_limut;
 821                 av->global_dest.flow_lable    = cv->cv_path.flow_label;
 822                 av->global_dest.s_gid_index   = cv->cv_sgid_index;
 823                 // XXX other av fields zero?
 824
 825                 rtr->destanation_qp            = cv->cv_remote_qpn;
 826                 rtr->receive_psn               = cv->cv_rxpsn;
 827                 rtr->responder_rdma_r_atom_num = IBNAL_OUS_DST_RD;
 828
 829                 // XXX ? rtr->opt_min_rnr_nak_timer = 16;
 830
 831
 832                 // XXX sdp sets VV_QP_AT_OP_F but no actual optional options
 833                 attr.modify.vv_qp_attr_mask = VV_QP_AT_ADD_VEC |
 834                                               VV_QP_AT_DEST_QP |
 835                                               VV_QP_AT_R_PSN |
 836                                               VV_QP_AT_MIN_RNR_NAK_T |
 837                                               VV_QP_AT_RESP_RDMA_ATOM_OUT_NUM |
 838                                               VV_QP_AT_OP_F;
 839                 break;
 840         }
 841         case vv_qp_state_rts: {
 842                 struct vv_qp_modify_rts_st *rts = &attr.modify.params.rts;
 843
 844                 rts->send_psn                 = cv->cv_txpsn;
 845                 rts->local_ack_timeout        = IBNAL_LOCAL_ACK_TIMEOUT;
 846                 rts->retry_num                = IBNAL_RETRY_CNT;
 847                 rts->rnr_num                  = IBNAL_RNR_CNT;
 848                 rts->dest_out_rdma_r_atom_num = IBNAL_OUS_DST_RD;
 849
 850                 attr.modify.vv_qp_attr_mask = VV_QP_AT_S_PSN |
 851                                               VV_QP_AT_L_ACK_T |
 852                                               VV_QP_AT_RETRY_NUM |
 853                                               VV_QP_AT_RNR_NUM |
 854                                               VV_QP_AT_DEST_RDMA_ATOM_OUT_NUM;
 855                 break;
 856         }
 857         case vv_qp_state_error:
 858         case vv_qp_state_reset:
 859                 attr.modify.vv_qp_attr_mask = 0;
 860                 break;
 861         }
 862
 863         attr.modify.qp_modify_into_state = new_state;
 864         attr.modify.vv_qp_attr_mask |= VV_QP_AT_STATE;
 865
 866         vvrc = vv_qp_modify(kibnal_data.kib_hca, conn->ibc_qp, &attr, NULL);
 867         if (vvrc != vv_return_ok) {
 868                 CERROR("Can't modify qp -> "LPX64" state to %d: %d\n",
 869                        conn->ibc_peer->ibp_nid, new_state, vvrc);
 870                 return -EIO;
 871         }
 872
 873         return 0;
 874 }
 875
 876 kib_conn_t *
 877 kibnal_create_conn (cm_cep_handle_t cep)
 878 {
 879         kib_conn_t   *conn;
 880         int           i;
 881         __u64         vaddr = 0;
 882         __u64         vaddr_base;
 883         int           page_offset;
 884         int           ipage;
 885         vv_return_t   vvrc;
 886         int           rc;
 887
 888         static vv_qp_attr_t  reqattr;
 889         static vv_qp_attr_t  rspattr;
 890
 891         /* Only the connd creates conns => single threaded */
 892         LASSERT(!in_interrupt());
 893         LASSERT(current == kibnal_data.kib_connd);
 894
 895         PORTAL_ALLOC(conn, sizeof (*conn));
 896         if (conn == NULL) {
 897                 CERROR ("Can't allocate connection\n");
 898                 return (NULL);
 899         }
 900
 901         /* zero flags, NULL pointers etc... */
 902         memset (conn, 0, sizeof (*conn));
 903
 904         INIT_LIST_HEAD (&conn->ibc_early_rxs);
 905         INIT_LIST_HEAD (&conn->ibc_tx_queue);
 906         INIT_LIST_HEAD (&conn->ibc_active_txs);
 907         spin_lock_init (&conn->ibc_lock);
 908
 909         atomic_inc (&kibnal_data.kib_nconns);
 910         /* well not really, but I call destroy() on failure, which decrements */
 911
 912         conn->ibc_cep = cep;
 913
 914         PORTAL_ALLOC(conn->ibc_connvars, sizeof(*conn->ibc_connvars));
 915         if (conn->ibc_connvars == NULL) {
 916                 CERROR("Can't allocate in-progress connection state\n");
 917                 goto failed;
 918         }
 919         memset (conn->ibc_connvars, 0, sizeof(*conn->ibc_connvars));
 920         /* Random seed for QP sequence number */
 921         get_random_bytes(&conn->ibc_connvars->cv_rxpsn,
 922                          sizeof(conn->ibc_connvars->cv_rxpsn));
 923
 924         PORTAL_ALLOC(conn->ibc_rxs, IBNAL_RX_MSGS * sizeof (kib_rx_t));
 925         if (conn->ibc_rxs == NULL) {
 926                 CERROR("Cannot allocate RX buffers\n");
 927                 goto failed;
 928         }
 929         memset (conn->ibc_rxs, 0, IBNAL_RX_MSGS * sizeof(kib_rx_t));
 930
 931         rc = kibnal_alloc_pages(&conn->ibc_rx_pages, IBNAL_RX_MSG_PAGES, 1);
 932         if (rc != 0)
 933                 goto failed;
 934
 935         vaddr_base = vaddr = conn->ibc_rx_pages->ibp_vaddr;
 936
 937         for (i = ipage = page_offset = 0; i < IBNAL_RX_MSGS; i++) {
 938                 struct page *page = conn->ibc_rx_pages->ibp_pages[ipage];
 939                 kib_rx_t   *rx = &conn->ibc_rxs[i];
 940
 941                 rx->rx_conn = conn;
 942                 rx->rx_msg = (kib_msg_t *)(((char *)page_address(page)) +
 943                              page_offset);
 944
 945 #if IBNAL_WHOLE_MEM
 946                 {
 947                         vv_mem_reg_h_t  mem_h;
 948                         vv_r_key_t      r_key;
 949
 950                         /* Voltaire stack already registers the whole
 951                          * memory, so use that API. */
 952                         vvrc = vv_get_gen_mr_attrib(kibnal_data.kib_hca,
 953                                                     rx->rx_msg,
 954                                                     IBNAL_MSG_SIZE,
 955                                                     &mem_h,
 956                                                     &rx->rx_lkey,
 957                                                     &r_key);
 958                         LASSERT (vvrc == vv_return_ok);
 959                 }
 960 #else
 961                 rx->rx_vaddr = vaddr;
 962 #endif
 963                 CDEBUG(D_NET, "Rx[%d] %p->%p[%x:"LPX64"]\n", i, rx,
 964                        rx->rx_msg, KIBNAL_RX_LKEY(rx), KIBNAL_RX_VADDR(rx));
 965
 966                 vaddr += IBNAL_MSG_SIZE;
 967                 LASSERT (vaddr <= vaddr_base + IBNAL_RX_MSG_BYTES);
 968
 969                 page_offset += IBNAL_MSG_SIZE;
 970                 LASSERT (page_offset <= PAGE_SIZE);
 971
 972                 if (page_offset == PAGE_SIZE) {
 973                         page_offset = 0;
 974                         ipage++;
 975                         LASSERT (ipage <= IBNAL_RX_MSG_PAGES);
 976                 }
 977         }
 978
 979         memset(&reqattr, 0, sizeof(reqattr));
 980
 981         reqattr.create.qp_type                    = vv_qp_type_r_conn;
 982         reqattr.create.cq_send_h                  = kibnal_data.kib_cq;
 983         reqattr.create.cq_receive_h               = kibnal_data.kib_cq;
 984         reqattr.create.send_max_outstand_wr       = (1 + IBNAL_MAX_RDMA_FRAGS) *
 985                                                     IBNAL_MSG_QUEUE_SIZE;
 986         reqattr.create.receive_max_outstand_wr    = IBNAL_RX_MSGS;
 987         reqattr.create.max_scatgat_per_send_wr    = 1;
 988         reqattr.create.max_scatgat_per_receive_wr = 1;
 989         reqattr.create.signaling_type             = vv_selectable_signaling;
 990         reqattr.create.pd_h                       = kibnal_data.kib_pd;
 991         reqattr.create.recv_solicited_events      = vv_selectable_signaling; // vv_signal_all;
 992
 993         vvrc = vv_qp_create(kibnal_data.kib_hca, &reqattr, NULL,
 994                             &conn->ibc_qp, &rspattr);
 995         if (vvrc != vv_return_ok) {
 996                 CERROR ("Failed to create queue pair: %d\n", vvrc);
 997                 goto failed;
 998         }
 999
1000         /* Mark QP created */
1001         conn->ibc_state = IBNAL_CONN_INIT_QP;
1002         conn->ibc_connvars->cv_local_qpn = rspattr.create_return.qp_num;
1003
1004         if (rspattr.create_return.receive_max_outstand_wr <
1005             IBNAL_MSG_QUEUE_SIZE ||
1006             rspattr.create_return.send_max_outstand_wr <
1007             (1 + IBNAL_MAX_RDMA_FRAGS) * IBNAL_MSG_QUEUE_SIZE) {
1008                 CERROR("Insufficient rx/tx work items: wanted %d/%d got %d/%d\n",
1009                        IBNAL_MSG_QUEUE_SIZE,
1010                        (1 + IBNAL_MAX_RDMA_FRAGS) * IBNAL_MSG_QUEUE_SIZE,
1011                        rspattr.create_return.receive_max_outstand_wr,
1012                        rspattr.create_return.send_max_outstand_wr);
1013                 goto failed;
1014         }
1015
1016         /* Mark init complete */
1017         conn->ibc_state = IBNAL_CONN_INIT;
1018
1019         /* 1 ref for caller */
1020         atomic_set (&conn->ibc_refcount, 1);
1021         return (conn);
1022
1023  failed:
1024         kibnal_destroy_conn (conn);
1025         return (NULL);
1026 }
1027
1028 void
1029 kibnal_destroy_conn (kib_conn_t *conn)
1030 {
1031         vv_return_t vvrc;
1032
1033         /* Only the connd does this (i.e. single threaded) */
1034         LASSERT (!in_interrupt());
1035         LASSERT (current == kibnal_data.kib_connd);
1036
1037         CDEBUG (D_NET, "connection %p\n", conn);
1038
1039         LASSERT (atomic_read (&conn->ibc_refcount) == 0);
1040         LASSERT (list_empty(&conn->ibc_early_rxs));
1041         LASSERT (list_empty(&conn->ibc_tx_queue));
1042         LASSERT (list_empty(&conn->ibc_active_txs));
1043         LASSERT (conn->ibc_nsends_posted == 0);
1044
1045         switch (conn->ibc_state) {
1046         default:
1047                 /* conn must be completely disengaged from the network */
1048                 LBUG();
1049
1050         case IBNAL_CONN_DISCONNECTED:
1051                 /* connvars should have been freed already */
1052                 LASSERT (conn->ibc_connvars == NULL);
1053                 /* fall through */
1054
1055         case IBNAL_CONN_INIT:
1056                 vvrc = cm_destroy_cep(conn->ibc_cep);
1057                 LASSERT (vvrc == vv_return_ok);
1058                 /* fall through */
1059
1060         case IBNAL_CONN_INIT_QP:
1061                 kibnal_set_qp_state(conn, vv_qp_state_reset);
1062                 vvrc = vv_qp_destroy(kibnal_data.kib_hca, conn->ibc_qp);
1063                 if (vvrc != vv_return_ok)
1064                         CERROR("Can't destroy QP: %d\n", vvrc);
1065                 /* fall through */
1066
1067         case IBNAL_CONN_INIT_NOTHING:
1068                 break;
1069         }
1070
1071         if (conn->ibc_rx_pages != NULL)
1072                 kibnal_free_pages(conn->ibc_rx_pages);
1073
1074         if (conn->ibc_rxs != NULL)
1075                 PORTAL_FREE(conn->ibc_rxs,
1076                             IBNAL_RX_MSGS * sizeof(kib_rx_t));
1077
1078         if (conn->ibc_connvars != NULL)
1079                 PORTAL_FREE(conn->ibc_connvars, sizeof(*conn->ibc_connvars));
1080
1081         if (conn->ibc_peer != NULL)
1082                 kibnal_peer_decref(conn->ibc_peer);
1083
1084         PORTAL_FREE(conn, sizeof (*conn));
1085
1086         atomic_dec(&kibnal_data.kib_nconns);
1087 }
1088
1089 int
1090 kibnal_close_peer_conns_locked (kib_peer_t *peer, int why)
1091 {
1092         kib_conn_t         *conn;
1093         struct list_head   *ctmp;
1094         struct list_head   *cnxt;
1095         int                 count = 0;
1096
1097         list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
1098                 conn = list_entry (ctmp, kib_conn_t, ibc_list);
1099
1100                 count++;
1101                 kibnal_close_conn_locked (conn, why);
1102         }
1103
1104         return (count);
1105 }
1106
1107 int
1108 kibnal_close_stale_conns_locked (kib_peer_t *peer, __u64 incarnation)
1109 {
1110         kib_conn_t         *conn;
1111         struct list_head   *ctmp;
1112         struct list_head   *cnxt;
1113         int                 count = 0;
1114
1115         list_for_each_safe (ctmp, cnxt, &peer->ibp_conns) {
1116                 conn = list_entry (ctmp, kib_conn_t, ibc_list);
1117
1118                 if (conn->ibc_incarnation == incarnation)
1119                         continue;
1120
1121                 CDEBUG(D_NET, "Closing stale conn nid:"LPX64" incarnation:"LPX64"("LPX64")\n",
1122                        peer->ibp_nid, conn->ibc_incarnation, incarnation);
1123
1124                 count++;
1125                 kibnal_close_conn_locked (conn, -ESTALE);
1126         }
1127
1128         return (count);
1129 }
1130
1131 int
1132 kibnal_close_matching_conns (ptl_nid_t nid)
1133 {
1134         kib_peer_t         *peer;
1135         struct list_head   *ptmp;
1136         struct list_head   *pnxt;
1137         int                 lo;
1138         int                 hi;
1139         int                 i;
1140         unsigned long       flags;
1141         int                 count = 0;
1142
1143         write_lock_irqsave(&kibnal_data.kib_global_lock, flags);
1144
1145         if (nid != PTL_NID_ANY)
1146                 lo = hi = kibnal_nid2peerlist(nid) - kibnal_data.kib_peers;
1147         else {
1148                 lo = 0;
1149                 hi = kibnal_data.kib_peer_hash_size - 1;
1150         }
1151
1152         for (i = lo; i <= hi; i++) {
1153                 list_for_each_safe (ptmp, pnxt, &kibnal_data.kib_peers[i]) {
1154
1155                         peer = list_entry (ptmp, kib_peer_t, ibp_list);
1156                         LASSERT (peer->ibp_persistence != 0 ||
1157                                  peer->ibp_connecting != 0 ||
1158                                  !list_empty (&peer->ibp_conns));
1159
1160                         if (!(nid == PTL_NID_ANY || nid == peer->ibp_nid))
1161                                 continue;
1162
1163                         count += kibnal_close_peer_conns_locked (peer, 0);
1164                 }
1165         }
1166
1167         write_unlock_irqrestore(&kibnal_data.kib_global_lock, flags);
1168
1169         /* wildcards always succeed */
1170         if (nid == PTL_NID_ANY)
1171                 return (0);
1172
1173         return (count == 0 ? -ENOENT : 0);
1174 }
1175
1176 int
1177 kibnal_cmd(struct portals_cfg *pcfg, void * private)
1178 {
1179         int rc = -EINVAL;
1180
1181         LASSERT (pcfg != NULL);
1182
1183         switch(pcfg->pcfg_command) {
1184         case NAL_CMD_GET_PEER: {
1185                 ptl_nid_t   nid = 0;
1186                 __u32       ip = 0;
1187                 int         share_count = 0;
1188
1189                 rc = kibnal_get_peer_info(pcfg->pcfg_count,
1190                                           &nid, &ip, &share_count);
1191                 pcfg->pcfg_nid   = nid;
1192                 pcfg->pcfg_size  = 0;
1193                 pcfg->pcfg_id    = ip;
1194                 pcfg->pcfg_misc  = IBNAL_SERVICE_NUMBER; /* port */
1195                 pcfg->pcfg_count = 0;
1196                 pcfg->pcfg_wait  = share_count;
1197                 break;
1198         }
1199         case NAL_CMD_ADD_PEER: {
1200                 rc = kibnal_add_persistent_peer (pcfg->pcfg_nid,
1201                                                  pcfg->pcfg_id); /* IP */
1202                 break;
1203         }
1204         case NAL_CMD_DEL_PEER: {
1205                 rc = kibnal_del_peer (pcfg->pcfg_nid,
1206                                        /* flags == single_share */
1207                                        pcfg->pcfg_flags != 0);
1208                 break;
1209         }
1210         case NAL_CMD_GET_CONN: {
1211                 kib_conn_t *conn = kibnal_get_conn_by_idx (pcfg->pcfg_count);
1212
1213                 if (conn == NULL)
1214                         rc = -ENOENT;
1215                 else {
1216                         rc = 0;
1217                         pcfg->pcfg_nid   = conn->ibc_peer->ibp_nid;
1218                         pcfg->pcfg_id    = 0;
1219                         pcfg->pcfg_misc  = 0;
1220                         pcfg->pcfg_flags = 0;
1221                         kibnal_conn_decref(conn);
1222                 }
1223                 break;
1224         }
1225         case NAL_CMD_CLOSE_CONNECTION: {
1226                 rc = kibnal_close_matching_conns (pcfg->pcfg_nid);
1227                 break;
1228         }
1229         case NAL_CMD_REGISTER_MYNID: {
1230                 if (pcfg->pcfg_nid == PTL_NID_ANY)
1231                         rc = -EINVAL;
1232                 else
1233                         rc = kibnal_set_mynid (pcfg->pcfg_nid);
1234                 break;
1235         }
1236         }
1237
1238         return rc;
1239 }
1240
1241 void
1242 kibnal_free_pages (kib_pages_t *p)
1243 {
1244         int         npages = p->ibp_npages;
1245         vv_return_t vvrc;
1246         int         i;
1247
1248         if (p->ibp_mapped) {
1249                 vvrc = vv_mem_region_destroy(kibnal_data.kib_hca,
1250                                              p->ibp_handle);
1251                 if (vvrc != vv_return_ok)
1252                         CERROR ("Deregister error: %d\n", vvrc);
1253         }
1254
1255         for (i = 0; i < npages; i++)
1256                 if (p->ibp_pages[i] != NULL)
1257                         __free_page(p->ibp_pages[i]);
1258
1259         PORTAL_FREE (p, offsetof(kib_pages_t, ibp_pages[npages]));
1260 }
1261
1262 int
1263 kibnal_alloc_pages (kib_pages_t **pp, int npages, int allow_write)
1264 {
1265         kib_pages_t   *p;
1266         int            i;
1267 #if !IBNAL_WHOLE_MEM
1268         vv_phy_list_t            vv_phys;
1269         vv_phy_buf_t            *phys_pages;
1270         vv_return_t              vvrc;
1271         vv_access_con_bit_mask_t access;
1272 #endif
1273
1274         PORTAL_ALLOC(p, offsetof(kib_pages_t, ibp_pages[npages]));
1275         if (p == NULL) {
1276                 CERROR ("Can't allocate buffer %d\n", npages);
1277                 return (-ENOMEM);
1278         }
1279
1280         memset (p, 0, offsetof(kib_pages_t, ibp_pages[npages]));
1281         p->ibp_npages = npages;
1282
1283         for (i = 0; i < npages; i++) {
1284                 p->ibp_pages[i] = alloc_page (GFP_KERNEL);
1285                 if (p->ibp_pages[i] == NULL) {
1286                         CERROR ("Can't allocate page %d of %d\n", i, npages);
1287                         kibnal_free_pages(p);
1288                         return (-ENOMEM);
1289                 }
1290         }
1291
1292 #if !IBNAL_WHOLE_MEM
1293         PORTAL_ALLOC(phys_pages, npages * sizeof(*phys_pages));
1294         if (phys_pages == NULL) {
1295                 CERROR ("Can't allocate physarray for %d pages\n", npages);
1296                 kibnal_free_pages(p);
1297                 return (-ENOMEM);
1298         }
1299
1300         vv_phys.number_of_buff = npages;
1301         vv_phys.phy_list = phys_pages;
1302
1303         for (i = 0; i < npages; i++) {
1304                 phys_pages[i].size = PAGE_SIZE;
1305                 phys_pages[i].start = kibnal_page2phys(p->ibp_pages[i]);
1306         }
1307
1308         VV_ACCESS_CONTROL_MASK_SET_ALL(access);
1309
1310         vvrc = vv_phy_mem_region_register(kibnal_data.kib_hca,
1311                                           &vv_phys,
1312                                           0, /* requested vaddr */
1313                                           npages * PAGE_SIZE, 0, /* offset */
1314                                           kibnal_data.kib_pd,
1315                                           access,
1316                                           &p->ibp_handle,
1317                                           &p->ibp_vaddr,
1318                                           &p->ibp_lkey,
1319                                           &p->ibp_rkey);
1320
1321         PORTAL_FREE(phys_pages, npages * sizeof(*phys_pages));
1322
1323         if (vvrc != vv_return_ok) {
1324                 CERROR ("Error %d mapping %d pages\n", vvrc, npages);
1325                 kibnal_free_pages(p);
1326                 return (-EFAULT);
1327         }
1328
1329         CDEBUG(D_NET, "registered %d pages; handle: %x vaddr "LPX64" "
1330                "lkey %x rkey %x\n", npages, p->ibp_handle,
1331                p->ibp_vaddr, p->ibp_lkey, p->ibp_rkey);
1332
1333         p->ibp_mapped = 1;
1334 #endif
1335         *pp = p;
1336         return (0);
1337 }
1338
1339 int
1340 kibnal_alloc_tx_descs (void)
1341 {
1342         int    i;
1343
1344         PORTAL_ALLOC (kibnal_data.kib_tx_descs,
1345                       IBNAL_TX_MSGS * sizeof(kib_tx_t));
1346         if (kibnal_data.kib_tx_descs == NULL)
1347                 return -ENOMEM;
1348
1349         memset(kibnal_data.kib_tx_descs, 0,
1350                IBNAL_TX_MSGS * sizeof(kib_tx_t));
1351
1352         for (i = 0; i < IBNAL_TX_MSGS; i++) {
1353                 kib_tx_t *tx = &kibnal_data.kib_tx_descs[i];
1354
1355                 PORTAL_ALLOC(tx->tx_wrq,
1356                              (1 + IBNAL_MAX_RDMA_FRAGS) *
1357                              sizeof(*tx->tx_wrq));
1358                 if (tx->tx_wrq == NULL)
1359                         return -ENOMEM;
1360
1361                 PORTAL_ALLOC(tx->tx_gl,
1362                              (1 + IBNAL_MAX_RDMA_FRAGS) *
1363                              sizeof(*tx->tx_gl));
1364                 if (tx->tx_gl == NULL)
1365                         return -ENOMEM;
1366
1367                 PORTAL_ALLOC(tx->tx_rd,
1368                              offsetof(kib_rdma_desc_t,
1369                                       rd_frags[IBNAL_MAX_RDMA_FRAGS]));
1370                 if (tx->tx_rd == NULL)
1371                         return -ENOMEM;
1372         }
1373
1374         return 0;
1375 }
1376
1377 void
1378 kibnal_free_tx_descs (void)
1379 {
1380         int    i;
1381
1382         if (kibnal_data.kib_tx_descs == NULL)
1383                 return;
1384
1385         for (i = 0; i < IBNAL_TX_MSGS; i++) {
1386                 kib_tx_t *tx = &kibnal_data.kib_tx_descs[i];
1387
1388                 if (tx->tx_wrq != NULL)
1389                         PORTAL_FREE(tx->tx_wrq,
1390                                     (1 + IBNAL_MAX_RDMA_FRAGS) *
1391                                     sizeof(*tx->tx_wrq));
1392
1393                 if (tx->tx_gl != NULL)
1394                         PORTAL_FREE(tx->tx_gl,
1395                                     (1 + IBNAL_MAX_RDMA_FRAGS) *
1396                                     sizeof(*tx->tx_gl));
1397
1398                 if (tx->tx_rd != NULL)
1399                         PORTAL_FREE(tx->tx_rd,
1400                                     offsetof(kib_rdma_desc_t,
1401                                              rd_frags[IBNAL_MAX_RDMA_FRAGS]));
1402         }
1403
1404         PORTAL_FREE(kibnal_data.kib_tx_descs,
1405                     IBNAL_TX_MSGS * sizeof(kib_tx_t));
1406 }
1407
1408 int
1409 kibnal_setup_tx_descs (void)
1410 {
1411         int           ipage = 0;
1412         int           page_offset = 0;
1413         __u64         vaddr;
1414         __u64         vaddr_base;
1415         struct page  *page;
1416         kib_tx_t     *tx;
1417         int           i;
1418         int           rc;
1419
1420         /* pre-mapped messages are not bigger than 1 page */
1421         CLASSERT (IBNAL_MSG_SIZE <= PAGE_SIZE);
1422
1423         /* No fancy arithmetic when we do the buffer calculations */
1424         CLASSERT (PAGE_SIZE % IBNAL_MSG_SIZE == 0);
1425
1426         rc = kibnal_alloc_pages(&kibnal_data.kib_tx_pages, IBNAL_TX_MSG_PAGES,
1427                                 0);
1428         if (rc != 0)
1429                 return (rc);
1430
1431         /* ignored for the whole_mem case */
1432         vaddr = vaddr_base = kibnal_data.kib_tx_pages->ibp_vaddr;
1433
1434         for (i = 0; i < IBNAL_TX_MSGS; i++) {
1435                 page = kibnal_data.kib_tx_pages->ibp_pages[ipage];
1436                 tx = &kibnal_data.kib_tx_descs[i];
1437
1438                 tx->tx_msg = (kib_msg_t *)(((char *)page_address(page)) +
1439                                            page_offset);
1440 #if IBNAL_WHOLE_MEM
1441                 {
1442                         vv_mem_reg_h_t  mem_h;
1443                         vv_r_key_t      rkey;
1444                         vv_return_t     vvrc;
1445
1446                         /* Voltaire stack already registers the whole
1447                          * memory, so use that API. */
1448                         vvrc = vv_get_gen_mr_attrib(kibnal_data.kib_hca,
1449                                                     tx->tx_msg,
1450                                                     IBNAL_MSG_SIZE,
1451                                                     &mem_h,
1452                                                     &tx->tx_lkey,
1453                                                     &rkey);
1454                         LASSERT (vvrc == vv_return_ok);
1455                 }
1456 #else
1457                 tx->tx_vaddr = vaddr;
1458 #endif
1459                 tx->tx_isnblk = (i >= IBNAL_NTX);
1460                 tx->tx_mapped = KIB_TX_UNMAPPED;
1461
1462                 CDEBUG(D_NET, "Tx[%d] %p->%p[%x:"LPX64"]\n", i, tx,
1463                        tx->tx_msg, KIBNAL_TX_LKEY(tx), KIBNAL_TX_VADDR(tx));
1464
1465                 if (tx->tx_isnblk)
1466                         list_add (&tx->tx_list,
1467                                   &kibnal_data.kib_idle_nblk_txs);
1468                 else
1469                         list_add (&tx->tx_list,
1470                                   &kibnal_data.kib_idle_txs);
1471
1472                 vaddr += IBNAL_MSG_SIZE;
1473                 LASSERT (vaddr <= vaddr_base + IBNAL_TX_MSG_BYTES);
1474
1475                 page_offset += IBNAL_MSG_SIZE;
1476                 LASSERT (page_offset <= PAGE_SIZE);
1477
1478                 if (page_offset == PAGE_SIZE) {
1479                         page_offset = 0;
1480                         ipage++;
1481                         LASSERT (ipage <= IBNAL_TX_MSG_PAGES);
1482                 }
1483         }
1484
1485         return (0);
1486 }
1487
1488 void
1489 kibnal_api_shutdown (nal_t *nal)
1490 {
1491         int         i;
1492         vv_return_t vvrc;
1493
1494         if (nal->nal_refct != 0) {
1495                 /* This module got the first ref */
1496                 PORTAL_MODULE_UNUSE;
1497                 return;
1498         }
1499
1500         CDEBUG(D_MALLOC, "before NAL cleanup: kmem %d\n",
1501                atomic_read (&portal_kmemory));
1502
1503         LASSERT(nal == &kibnal_api);
1504
1505         switch (kibnal_data.kib_init) {
1506
1507         case IBNAL_INIT_ALL:
1508                 /* stop calls to nal_cmd */
1509                 libcfs_nal_cmd_unregister(VIBNAL);
1510                 /* No new peers */
1511
1512                 /* resetting my NID removes my listener and nukes all current
1513                  * peers and their connections */
1514                 kibnal_set_mynid (PTL_NID_ANY);
1515
1516                 /* Wait for all peer state to clean up */
1517                 i = 2;
1518                 while (atomic_read (&kibnal_data.kib_npeers) != 0) {
1519                         i++;
1520                         CDEBUG(((i & (-i)) == i) ? D_WARNING : D_NET, /* power of 2? */
1521                                "waiting for %d peers to disconnect\n",
1522                                atomic_read (&kibnal_data.kib_npeers));
1523                         set_current_state (TASK_UNINTERRUPTIBLE);
1524                         schedule_timeout (HZ);
1525                 }
1526                 /* fall through */
1527
1528         case IBNAL_INIT_CQ:
1529                 vvrc = vv_cq_destroy(kibnal_data.kib_hca, kibnal_data.kib_cq);
1530                 if (vvrc != vv_return_ok)
1531                         CERROR ("Destroy CQ error: %d\n", vvrc);
1532                 /* fall through */
1533
1534         case IBNAL_INIT_TXD:
1535                 kibnal_free_pages (kibnal_data.kib_tx_pages);
1536                 /* fall through */
1537
1538         case IBNAL_INIT_PD:
1539 #if !IBNAL_WHOLE_MEM
1540                 vvrc = vv_pd_deallocate(kibnal_data.kib_hca,
1541                                         kibnal_data.kib_pd);
1542                 if (vvrc != vv_return_ok)
1543                         CERROR ("Destroy PD error: %d\n", vvrc);
1544 #endif
1545                 /* fall through */
1546
1547         case IBNAL_INIT_ASYNC:
1548                 vvrc = vv_dell_async_event_cb (kibnal_data.kib_hca,
1549                                               kibnal_async_callback);
1550                 if (vvrc != vv_return_ok)
1551                         CERROR("vv_dell_async_event_cb error: %d\n", vvrc);
1552
1553                 /* fall through */
1554
1555         case IBNAL_INIT_HCA:
1556                 vvrc = vv_hca_close(kibnal_data.kib_hca);
1557                 if (vvrc != vv_return_ok)
1558                         CERROR ("Close HCA  error: %d\n", vvrc);
1559                 /* fall through */
1560
1561         case IBNAL_INIT_LIB:
1562                 lib_fini(&kibnal_lib);
1563                 /* fall through */
1564
1565         case IBNAL_INIT_DATA:
1566                 LASSERT (atomic_read (&kibnal_data.kib_npeers) == 0);
1567                 LASSERT (kibnal_data.kib_peers != NULL);
1568                 for (i = 0; i < kibnal_data.kib_peer_hash_size; i++) {
1569                         LASSERT (list_empty (&kibnal_data.kib_peers[i]));
1570                 }
1571                 LASSERT (atomic_read (&kibnal_data.kib_nconns) == 0);
1572                 LASSERT (list_empty (&kibnal_data.kib_sched_rxq));
1573                 LASSERT (list_empty (&kibnal_data.kib_sched_txq));
1574                 LASSERT (list_empty (&kibnal_data.kib_connd_zombies));
1575                 LASSERT (list_empty (&kibnal_data.kib_connd_conns));
1576                 LASSERT (list_empty (&kibnal_data.kib_connd_pcreqs));
1577                 LASSERT (list_empty (&kibnal_data.kib_connd_peers));
1578
1579                 /* flag threads to terminate; wake and wait for them to die */
1580                 kibnal_data.kib_shutdown = 1;
1581                 wake_up_all (&kibnal_data.kib_sched_waitq);
1582                 wake_up_all (&kibnal_data.kib_connd_waitq);
1583
1584                 i = 2;
1585                 while (atomic_read (&kibnal_data.kib_nthreads) != 0) {
1586                         i++;
1587                         CDEBUG(((i & (-i)) == i) ? D_WARNING : D_NET, /* power of 2? */
1588                                "Waiting for %d threads to terminate\n",
1589                                atomic_read (&kibnal_data.kib_nthreads));
1590                         set_current_state (TASK_INTERRUPTIBLE);
1591                         schedule_timeout (HZ);
1592                 }
1593                 /* fall through */
1594
1595         case IBNAL_INIT_NOTHING:
1596                 break;
1597         }
1598
1599         kibnal_free_tx_descs();
1600
1601         if (kibnal_data.kib_peers != NULL)
1602                 PORTAL_FREE (kibnal_data.kib_peers,
1603                              sizeof (struct list_head) *
1604                              kibnal_data.kib_peer_hash_size);
1605
1606         CDEBUG(D_MALLOC, "after NAL cleanup: kmem %d\n",
1607                atomic_read (&portal_kmemory));
1608         printk(KERN_INFO "Lustre: Voltaire IB NAL unloaded (final mem %d)\n",
1609                atomic_read(&portal_kmemory));
1610
1611         kibnal_data.kib_init = IBNAL_INIT_NOTHING;
1612 }
1613
1614 int
1615 kibnal_api_startup (nal_t *nal, ptl_pid_t requested_pid,
1616                      ptl_ni_limits_t *requested_limits,
1617                      ptl_ni_limits_t *actual_limits)
1618 {
1619         struct timeval            tv;
1620         ptl_process_id_t          process_id;
1621         int                       pkmem = atomic_read(&portal_kmemory);
1622         int                       rc;
1623         int                       i;
1624         vv_request_event_record_t req_er;
1625         vv_return_t               vvrc;
1626
1627         LASSERT (nal == &kibnal_api);
1628
1629         if (nal->nal_refct != 0) {
1630                 if (actual_limits != NULL)
1631                         *actual_limits = kibnal_lib.libnal_ni.ni_actual_limits;
1632                 /* This module got the first ref */
1633                 PORTAL_MODULE_USE;
1634                 return (PTL_OK);
1635         }
1636
1637         LASSERT (kibnal_data.kib_init == IBNAL_INIT_NOTHING);
1638         memset (&kibnal_data, 0, sizeof (kibnal_data)); /* zero pointers, flags etc */
1639
1640         do_gettimeofday(&tv);
1641         kibnal_data.kib_incarnation = (((__u64)tv.tv_sec) * 1000000) + tv.tv_usec;
1642         kibnal_data.kib_svc_id = IBNAL_SERVICE_NUMBER;
1643
1644         init_MUTEX (&kibnal_data.kib_nid_mutex);
1645
1646         rwlock_init(&kibnal_data.kib_global_lock);
1647
1648         kibnal_data.kib_peer_hash_size = IBNAL_PEER_HASH_SIZE;
1649         PORTAL_ALLOC (kibnal_data.kib_peers,
1650                       sizeof (struct list_head) * kibnal_data.kib_peer_hash_size);
1651         if (kibnal_data.kib_peers == NULL) {
1652                 goto failed;
1653         }
1654         for (i = 0; i < kibnal_data.kib_peer_hash_size; i++)
1655                 INIT_LIST_HEAD(&kibnal_data.kib_peers[i]);
1656
1657         spin_lock_init (&kibnal_data.kib_connd_lock);
1658         INIT_LIST_HEAD (&kibnal_data.kib_connd_peers);
1659         INIT_LIST_HEAD (&kibnal_data.kib_connd_pcreqs);
1660         INIT_LIST_HEAD (&kibnal_data.kib_connd_conns);
1661         INIT_LIST_HEAD (&kibnal_data.kib_connd_zombies);
1662         init_waitqueue_head (&kibnal_data.kib_connd_waitq);
1663
1664         spin_lock_init (&kibnal_data.kib_sched_lock);
1665         INIT_LIST_HEAD (&kibnal_data.kib_sched_txq);
1666         INIT_LIST_HEAD (&kibnal_data.kib_sched_rxq);
1667         init_waitqueue_head (&kibnal_data.kib_sched_waitq);
1668
1669         spin_lock_init (&kibnal_data.kib_tx_lock);
1670         INIT_LIST_HEAD (&kibnal_data.kib_idle_txs);
1671         INIT_LIST_HEAD (&kibnal_data.kib_idle_nblk_txs);
1672         init_waitqueue_head(&kibnal_data.kib_idle_tx_waitq);
1673
1674         rc = kibnal_alloc_tx_descs();
1675         if (rc != 0) {
1676                 CERROR("Can't allocate tx descs\n");
1677                 goto failed;
1678         }
1679
1680         /* lists/ptrs/locks initialised */
1681         kibnal_data.kib_init = IBNAL_INIT_DATA;
1682         /*****************************************************/
1683
1684         process_id.pid = requested_pid;
1685         process_id.nid = PTL_NID_ANY;
1686
1687         rc = lib_init(&kibnal_lib, nal, process_id,
1688                       requested_limits, actual_limits);
1689         if (rc != PTL_OK) {
1690                 CERROR("lib_init failed: error %d\n", rc);
1691                 goto failed;
1692         }
1693
1694         /* lib interface initialised */
1695         kibnal_data.kib_init = IBNAL_INIT_LIB;
1696         /*****************************************************/
1697
1698         for (i = 0; i < IBNAL_N_SCHED; i++) {
1699                 rc = kibnal_thread_start (kibnal_scheduler, (void *)((long)i));
1700                 if (rc != 0) {
1701                         CERROR("Can't spawn vibnal scheduler[%d]: %d\n",
1702                                i, rc);
1703                         goto failed;
1704                 }
1705         }
1706
1707         rc = kibnal_thread_start (kibnal_connd, NULL);
1708         if (rc != 0) {
1709                 CERROR ("Can't spawn vibnal connd: %d\n", rc);
1710                 goto failed;
1711         }
1712
1713         /* TODO: apparently only one adapter is supported */
1714         vvrc = vv_hca_open("InfiniHost0", NULL, &kibnal_data.kib_hca);
1715         if (vvrc != vv_return_ok) {
1716                 CERROR ("Can't open CA: %d\n", vvrc);
1717                 goto failed;
1718         }
1719
1720         /* Channel Adapter opened */
1721         kibnal_data.kib_init = IBNAL_INIT_HCA;
1722
1723         /* register to get HCA's asynchronous events. */
1724         req_er.req_event_type = VV_ASYNC_EVENT_ALL_MASK;
1725         vvrc = vv_set_async_event_cb (kibnal_data.kib_hca, req_er,
1726                                      kibnal_async_callback);
1727         if (vvrc != vv_return_ok) {
1728                 CERROR ("Can't open CA: %d\n", vvrc);
1729                 goto failed;
1730         }
1731
1732         kibnal_data.kib_init = IBNAL_INIT_ASYNC;
1733
1734         /*****************************************************/
1735
1736         vvrc = vv_hca_query(kibnal_data.kib_hca, &kibnal_data.kib_hca_attrs);
1737         if (vvrc != vv_return_ok) {
1738                 CERROR ("Can't size port attrs: %d\n", vvrc);
1739                 goto failed;
1740         }
1741
1742         kibnal_data.kib_port = -1;
1743
1744         for (i = 0; i<kibnal_data.kib_hca_attrs.port_num; i++) {
1745
1746                 int port_num = i+1;
1747                 u_int32_t tbl_count;
1748                 vv_port_attrib_t *pattr = &kibnal_data.kib_port_attr;
1749
1750                 vvrc = vv_port_query(kibnal_data.kib_hca, port_num, pattr);
1751                 if (vvrc != vv_return_ok) {
1752                         CERROR("vv_port_query failed for port %d: %d\n",
1753                                port_num, vvrc);
1754                         continue;
1755                 }
1756
1757                 switch (pattr->port_state) {
1758                 case vv_state_linkDoun:
1759                         CDEBUG(D_NET, "port[%d] Down\n", port_num);
1760                         continue;
1761                 case vv_state_linkInit:
1762                         CDEBUG(D_NET, "port[%d] Init\n", port_num);
1763                         continue;
1764                 case vv_state_linkArm:
1765                         CDEBUG(D_NET, "port[%d] Armed\n", port_num);
1766                         continue;
1767                 case vv_state_linkActive:
1768                         CDEBUG(D_NET, "port[%d] Active\n", port_num);
1769
1770                         /* Found a suitable port. Get its GUID and PKEY. */
1771                         kibnal_data.kib_port = port_num;
1772
1773                         tbl_count = 1;
1774                         vvrc = vv_get_port_gid_tbl(kibnal_data.kib_hca,
1775                                                    port_num, &tbl_count,
1776                                                    &kibnal_data.kib_port_gid);
1777                         if (vvrc != vv_return_ok) {
1778                                 CERROR("vv_get_port_gid_tbl failed "
1779                                        "for port %d: %d\n", port_num, vvrc);
1780                                 continue;
1781                         }
1782
1783                         tbl_count = 1;
1784                         vvrc = vv_get_port_partition_tbl(kibnal_data.kib_hca,
1785                                                         port_num, &tbl_count,
1786                                                         &kibnal_data.kib_port_pkey);
1787                         if (vvrc != vv_return_ok) {
1788                                 CERROR("vv_get_port_partition_tbl failed "
1789                                        "for port %d: %d\n", port_num, vvrc);
1790                                 continue;
1791                         }
1792
1793                         break;
1794                 case vv_state_linkActDefer: /* TODO: correct? */
1795                 case vv_state_linkNoChange:
1796                         CERROR("Unexpected port[%d] state %d\n",
1797                                i, pattr->port_state);
1798                         continue;
1799                 }
1800                 break;
1801         }
1802
1803         if (kibnal_data.kib_port == -1) {
1804                 CERROR ("Can't find an active port\n");
1805                 goto failed;
1806         }
1807
1808         CDEBUG(D_NET, "Using port %d - GID="LPX64":"LPX64"\n",
1809                kibnal_data.kib_port,
1810                kibnal_data.kib_port_gid.scope.g.subnet,
1811                kibnal_data.kib_port_gid.scope.g.eui64);
1812
1813         /*****************************************************/
1814
1815 #if !IBNAL_WHOLE_MEM
1816         vvrc = vv_pd_allocate(kibnal_data.kib_hca, &kibnal_data.kib_pd);
1817 #else
1818         vvrc = vv_get_gen_pd_h(kibnal_data.kib_hca, &kibnal_data.kib_pd);
1819 #endif
1820         if (vvrc != 0) {
1821                 CERROR ("Can't create PD: %d\n", vvrc);
1822                 goto failed;
1823         }
1824
1825         /* flag PD initialised */
1826         kibnal_data.kib_init = IBNAL_INIT_PD;
1827         /*****************************************************/
1828
1829         rc = kibnal_setup_tx_descs();
1830         if (rc != 0) {
1831                 CERROR ("Can't register tx descs: %d\n", rc);
1832                 goto failed;
1833         }
1834
1835         /* flag TX descs initialised */
1836         kibnal_data.kib_init = IBNAL_INIT_TXD;
1837         /*****************************************************/
1838         {
1839                 uint32_t nentries;
1840
1841                 vvrc = vv_cq_create(kibnal_data.kib_hca, IBNAL_CQ_ENTRIES,
1842                                     kibnal_cq_callback,
1843                                     NULL, /* context */
1844                                     &kibnal_data.kib_cq, &nentries);
1845                 if (vvrc != 0) {
1846                         CERROR ("Can't create RX CQ: %d\n", vvrc);
1847                         goto failed;
1848                 }
1849
1850                 /* flag CQ initialised */
1851                 kibnal_data.kib_init = IBNAL_INIT_CQ;
1852
1853                 if (nentries < IBNAL_CQ_ENTRIES) {
1854                         CERROR ("CQ only has %d entries, need %d\n",
1855                                 nentries, IBNAL_CQ_ENTRIES);
1856                         goto failed;
1857                 }
1858
1859                 vvrc = vv_request_completion_notification(kibnal_data.kib_hca,
1860                                                           kibnal_data.kib_cq,
1861                                                           vv_next_solicit_unsolicit_event);
1862                 if (vvrc != 0) {
1863                         CERROR ("Failed to re-arm completion queue: %d\n", rc);
1864                         goto failed;
1865                 }
1866         }
1867
1868         /*****************************************************/
1869
1870         rc = libcfs_nal_cmd_register(VIBNAL, &kibnal_cmd, NULL);
1871         if (rc != 0) {
1872                 CERROR ("Can't initialise command interface (rc = %d)\n", rc);
1873                 goto failed;
1874         }
1875
1876         /* flag everything initialised */
1877         kibnal_data.kib_init = IBNAL_INIT_ALL;
1878         /*****************************************************/
1879
1880         printk(KERN_INFO "Lustre: Voltaire IB NAL loaded "
1881                "(initial mem %d)\n", pkmem);
1882
1883         return (PTL_OK);
1884
1885  failed:
1886         CDEBUG(D_NET, "kibnal_api_startup failed\n");
1887         kibnal_api_shutdown (&kibnal_api);
1888         return (PTL_FAIL);
1889 }
1890
1891 void __exit
1892 kibnal_module_fini (void)
1893 {
1894 #ifdef CONFIG_SYSCTL
1895         if (kibnal_tunables.kib_sysctl != NULL)
1896                 unregister_sysctl_table (kibnal_tunables.kib_sysctl);
1897 #endif
1898         PtlNIFini(kibnal_ni);
1899
1900         ptl_unregister_nal(VIBNAL);
1901 }
1902
1903 int __init
1904 kibnal_module_init (void)
1905 {
1906         int    rc;
1907
1908         vibnal_assert_wire_constants();
1909
1910         CLASSERT (offsetof(kib_msg_t, ibm_u) + sizeof(kib_connparams_t)
1911                   <= cm_REQ_priv_data_len);
1912         CLASSERT (offsetof(kib_msg_t, ibm_u) + sizeof(kib_connparams_t)
1913                   <= cm_REP_priv_data_len);
1914         CLASSERT (offsetof(kib_msg_t, ibm_u.get.ibgm_rd.rd_frags[IBNAL_MAX_RDMA_FRAGS])
1915                   <= IBNAL_MSG_SIZE);
1916         CLASSERT (offsetof(kib_msg_t, ibm_u.putack.ibpam_rd.rd_frags[IBNAL_MAX_RDMA_FRAGS])
1917                   <= IBNAL_MSG_SIZE);
1918
1919         /* the following must be sizeof(int) for proc_dointvec() */
1920         CLASSERT (sizeof (kibnal_tunables.kib_io_timeout) == sizeof (int));
1921
1922         kibnal_api.nal_ni_init = kibnal_api_startup;
1923         kibnal_api.nal_ni_fini = kibnal_api_shutdown;
1924
1925         /* Initialise dynamic tunables to defaults once only */
1926         kibnal_tunables.kib_io_timeout = IBNAL_IO_TIMEOUT;
1927
1928         rc = ptl_register_nal(VIBNAL, &kibnal_api);
1929         if (rc != PTL_OK) {
1930                 CERROR("Can't register IBNAL: %d\n", rc);
1931                 return (-ENOMEM);               /* or something... */
1932         }
1933
1934         /* Pure gateways want the NAL started up at module load time... */
1935         rc = PtlNIInit(VIBNAL, LUSTRE_SRV_PTL_PID, NULL, NULL, &kibnal_ni);
1936         if (rc != PTL_OK && rc != PTL_IFACE_DUP) {
1937                 ptl_unregister_nal(VIBNAL);
1938                 return (-ENODEV);
1939         }
1940
1941 #ifdef CONFIG_SYSCTL
1942         /* Press on regardless even if registering sysctl doesn't work */
1943         kibnal_tunables.kib_sysctl =
1944                 register_sysctl_table (kibnal_top_ctl_table, 0);
1945 #endif
1946         return (0);
1947 }
1948
1949 MODULE_AUTHOR("Cluster File Systems, Inc. <info@clusterfs.com>");
1950 MODULE_DESCRIPTION("Kernel Voltaire IB NAL v0.01");
1951 MODULE_LICENSE("GPL");
1952
1953 module_init(kibnal_module_init);
1954 module_exit(kibnal_module_fini);
1955