lustre/portals/knals/iibnal/iibnal.h

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * Copyright (C) 2004 Cluster File Systems, Inc.
   5  *   Author: Eric Barton <eric@bartonsoftware.com>
   6  *
   7  *   This file is part of Lustre, http://www.lustre.org.
   8  *
   9  *   Lustre is free software; you can redistribute it and/or
  10  *   modify it under the terms of version 2 of the GNU General Public
  11  *   License as published by the Free Software Foundation.
  12  *
  13  *   Lustre is distributed in the hope that it will be useful,
  14  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  *   GNU General Public License for more details.
  17  *
  18  *   You should have received a copy of the GNU General Public License
  19  *   along with Lustre; if not, write to the Free Software
  20  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  21  *
  22  */
  23
  24 #ifndef EXPORT_SYMTAB
  25 # define EXPORT_SYMTAB
  26 #endif
  27
  28 #include <linux/config.h>
  29 #include <linux/module.h>
  30 #include <linux/kernel.h>
  31 #include <linux/mm.h>
  32 #include <linux/string.h>
  33 #include <linux/stat.h>
  34 #include <linux/errno.h>
  35 #include <linux/smp_lock.h>
  36 #include <linux/unistd.h>
  37 #include <linux/uio.h>
  38
  39 #include <asm/system.h>
  40 #include <asm/uaccess.h>
  41 #include <asm/io.h>
  42
  43 #include <linux/init.h>
  44 #include <linux/fs.h>
  45 #include <linux/file.h>
  46 #include <linux/stat.h>
  47 #include <linux/list.h>
  48 #include <linux/kmod.h>
  49 #include <linux/sysctl.h>
  50
  51 #define DEBUG_SUBSYSTEM S_NAL
  52
  53 #include <linux/kp30.h>
  54 #include <portals/p30.h>
  55 #include <portals/lib-p30.h>
  56 #include <portals/nal.h>
  57
  58 #include <linux/iba/ibt.h>
  59
  60 #define GCC_VERSION (__GNUC__ * 10000 \
  61                 + __GNUC_MINOR__ * 100 \
  62                 + __GNUC_PATCHLEVEL__)
  63
  64 /* Test for GCC > 3.2.2 */
  65 #if GCC_VERSION <= 30202
  66 /* GCC 3.2.2, and presumably several versions before it, will
  67  * miscompile this driver. See
  68  * http://gcc.gnu.org/bugzilla/show_bug.cgi?id=9853. */
  69 #error Invalid GCC version. Must use GCC >= 3.2.3
  70 #endif
  71
  72 #define IBNAL_SERVICE_NAME   "iibnal"
  73 #define IBNAL_SERVICE_NUMBER 0x11b9a1
  74
  75 #if CONFIG_SMP
  76 # define IBNAL_N_SCHED      num_online_cpus()   /* # schedulers */
  77 #else
  78 # define IBNAL_N_SCHED      1                   /* # schedulers */
  79 #endif
  80
  81 #define IBNAL_MIN_RECONNECT_INTERVAL HZ         /* first failed connection retry... */
  82 #define IBNAL_MAX_RECONNECT_INTERVAL (60*HZ)    /* ...exponentially increasing to this */
  83
  84 #define IBNAL_MSG_SIZE       (4<<10)            /* max size of queued messages (inc hdr) */
  85
  86 #define IBNAL_MSG_QUEUE_SIZE   8                /* # messages/RDMAs in-flight */
  87 #define IBNAL_CREDIT_HIGHWATER 7                /* when to eagerly return credits */
  88 /* 7 indicates infinite retry attempts, Infinicon recommended 5 */
  89 #define IBNAL_RETRY            5                /* # times to retry */
  90 #define IBNAL_RNR_RETRY        5                /*  */
  91 #define IBNAL_CM_RETRY         5                /* # times to retry connection */
  92 #define IBNAL_FLOW_CONTROL     1
  93 #define IBNAL_ACK_TIMEOUT       20              /* supposedly 4 secs */
  94
  95 #define IBNAL_NTX             64                /* # tx descs */
  96 /* this had to be dropped down so that we only register < 255 pages per
  97  * region.  this will change if we register all memory. */
  98 #define IBNAL_NTX_NBLK        128               /* # reserved tx descs */
  99
 100 #define IBNAL_PEER_HASH_SIZE  101               /* # peer lists */
 101
 102 #define IBNAL_RESCHED         100               /* # scheduler loops before reschedule */
 103
 104 #define IBNAL_CONCURRENT_PEERS 1000             /* # nodes all talking at once to me */
 105
 106 /* default vals for runtime tunables */
 107 #define IBNAL_IO_TIMEOUT      50                /* default comms timeout (seconds) */
 108
 109 /************************/
 110 /* derived constants... */
 111
 112 /* TX messages (shared by all connections) */
 113 #define IBNAL_TX_MSGS       (IBNAL_NTX + IBNAL_NTX_NBLK)
 114 #define IBNAL_TX_MSG_BYTES  (IBNAL_TX_MSGS * IBNAL_MSG_SIZE)
 115 #define IBNAL_TX_MSG_PAGES  ((IBNAL_TX_MSG_BYTES + PAGE_SIZE - 1)/PAGE_SIZE)
 116
 117 #define IBNAL_TX_MAX_SG (PTL_MD_MAX_IOV + 1)
 118
 119 /* RX messages (per connection) */
 120 #define IBNAL_RX_MSGS       IBNAL_MSG_QUEUE_SIZE
 121 #define IBNAL_RX_MSG_BYTES  (IBNAL_RX_MSGS * IBNAL_MSG_SIZE)
 122 #define IBNAL_RX_MSG_PAGES  ((IBNAL_RX_MSG_BYTES + PAGE_SIZE - 1)/PAGE_SIZE)
 123
 124
 125 /* we may have up to 2 completions per transmit +
 126    1 completion per receive, per connection */
 127 #define IBNAL_CQ_ENTRIES  ((2*IBNAL_TX_MSGS) +                          \
 128                            (IBNAL_RX_MSGS * IBNAL_CONCURRENT_PEERS))
 129
 130 #define IBNAL_RDMA_BASE  0x0eeb0000
 131 #define IBNAL_FMR        0
 132 #define IBNAL_WHOLE_MEM  1
 133 #define IBNAL_CKSUM      0
 134 //#define IBNAL_CALLBACK_CTXT  IB_CQ_CALLBACK_PROCESS
 135 #define IBNAL_CALLBACK_CTXT  IB_CQ_CALLBACK_INTERRUPT
 136
 137 /* XXX I have no idea. */
 138 #define IBNAL_STARTING_PSN 1
 139
 140 typedef struct
 141 {
 142         int               kib_io_timeout;       /* comms timeout (seconds) */
 143         struct ctl_table_header *kib_sysctl;    /* sysctl interface */
 144 } kib_tunables_t;
 145
 146 /* some of these have specific types in the stack that just map back
 147  * to the uFOO types, like IB_{L,R}_KEY. */
 148 typedef struct
 149 {
 150         int               ibp_npages;           /* # pages */
 151         int               ibp_mapped;           /* mapped? */
 152         __u64             ibp_vaddr;            /* mapped region vaddr */
 153         __u32             ibp_lkey;             /* mapped region lkey */
 154         __u32             ibp_rkey;             /* mapped region rkey */
 155         IB_HANDLE         ibp_handle;           /* mapped region handle */
 156         struct page      *ibp_pages[0];
 157 } kib_pages_t;
 158
 159 typedef struct
 160 {
 161         IB_HANDLE         md_handle;
 162         __u32             md_lkey;
 163         __u32             md_rkey;
 164         __u64             md_addr;
 165 } kib_md_t __attribute__((packed));
 166
 167 typedef struct
 168 {
 169         int               kib_init;             /* initialisation state */
 170         __u64             kib_incarnation;      /* which one am I */
 171         int               kib_shutdown;         /* shut down? */
 172         atomic_t          kib_nthreads;         /* # live threads */
 173
 174         __u64             kib_service_id;       /* service number I listen on */
 175         __u64             kib_port_guid;        /* my GUID (lo 64 of GID)*/
 176         __u16             kib_port_pkey;        /* my pkey, whatever that is */
 177         ptl_nid_t         kib_nid;              /* my NID */
 178         struct semaphore  kib_nid_mutex;        /* serialise NID ops */
 179         struct semaphore  kib_nid_signal;       /* signal completion */
 180         IB_HANDLE         kib_cep;              /* connection end point */
 181
 182         rwlock_t          kib_global_lock;      /* stabilize peer/conn ops */
 183
 184         struct list_head *kib_peers;            /* hash table of all my known peers */
 185         int               kib_peer_hash_size;   /* size of kib_peers */
 186         atomic_t          kib_npeers;           /* # peers extant */
 187         atomic_t          kib_nconns;           /* # connections extant */
 188
 189         struct list_head  kib_connd_conns;      /* connections to progress */
 190         struct list_head  kib_connd_peers;      /* peers waiting for a connection */
 191         wait_queue_head_t kib_connd_waitq;      /* connection daemons sleep here */
 192         unsigned long     kib_connd_waketime;   /* when connd will wake */
 193         spinlock_t        kib_connd_lock;       /* serialise */
 194
 195         wait_queue_head_t kib_sched_waitq;      /* schedulers sleep here */
 196         struct list_head  kib_sched_txq;        /* tx requiring attention */
 197         struct list_head  kib_sched_rxq;        /* rx requiring attention */
 198         spinlock_t        kib_sched_lock;       /* serialise */
 199
 200         struct kib_tx    *kib_tx_descs;         /* all the tx descriptors */
 201         kib_pages_t      *kib_tx_pages;         /* premapped tx msg pages */
 202
 203         struct list_head  kib_idle_txs;         /* idle tx descriptors */
 204         struct list_head  kib_idle_nblk_txs;    /* idle reserved tx descriptors */
 205         wait_queue_head_t kib_idle_tx_waitq;    /* block here for tx descriptor */
 206         __u64             kib_next_tx_cookie;   /* RDMA completion cookie */
 207         spinlock_t        kib_tx_lock;          /* serialise */
 208
 209         IB_HANDLE         kib_hca;              /* The HCA */
 210         int               kib_port;             /* port on the device */
 211         IB_HANDLE         kib_pd;               /* protection domain */
 212         IB_HANDLE         kib_sd;               /* SD handle */
 213         IB_HANDLE         kib_cq;               /* completion queue */
 214         kib_md_t          kib_md;               /* full-mem registration */
 215
 216         void             *kib_listen_handle;    /* where I listen for connections */
 217
 218         IBT_INTERFACE_UNION kib_interfaces;     /* The Infinicon IBT interface */
 219
 220         uint64              kib_hca_guids[8];   /* all the HCA guids */
 221         IB_CA_ATTRIBUTES    kib_hca_attrs;      /* where to get HCA attrs */
 222         FABRIC_OPERATION_DATA kib_fabopdata;    /* (un)advertise service record */
 223 } kib_data_t;
 224
 225 #define IBNAL_INIT_NOTHING         0
 226 #define IBNAL_INIT_DATA            1
 227 #define IBNAL_INIT_LIB             2
 228 #define IBNAL_INIT_HCA             3
 229 #define IBNAL_INIT_PORTATTRS       4
 230 #define IBNAL_INIT_PORT            5
 231 #define IBNAL_INIT_SD              6
 232 #define IBNAL_INIT_PD              7
 233 #define IBNAL_INIT_FMR             8
 234 #define IBNAL_INIT_MR              9
 235 #define IBNAL_INIT_TXD             10
 236 #define IBNAL_INIT_CQ              11
 237 #define IBNAL_INIT_ALL             12
 238
 239 /************************************************************************
 240  * Wire message structs.
 241  * These are sent in sender's byte order (i.e. receiver flips).
 242  * CAVEAT EMPTOR: other structs communicated between nodes (e.g. MAD
 243  * private data and SM service info), is LE on the wire.
 244  */
 245
 246 /* also kib_md_t above */
 247
 248 typedef struct
 249 {
 250         __u32                 rd_key;           /* remote key */
 251         __u32                 rd_nob;           /* # of bytes */
 252         __u64                 rd_addr;          /* remote io vaddr */
 253 } kib_rdma_desc_t __attribute__((packed));
 254
 255 typedef struct
 256 {
 257         ptl_hdr_t         ibim_hdr;             /* portals header */
 258         char              ibim_payload[0];      /* piggy-backed payload */
 259 } kib_immediate_msg_t __attribute__((packed));
 260
 261 /* these arrays serve two purposes during rdma.  they are built on the passive
 262  * side and sent to the active side as remote arguments.  On the active side
 263  * the descs are used as a data structure on the way to local gather items.
 264  * the different roles result in split local/remote meaning of desc->rd_key */
 265 typedef struct
 266 {
 267         ptl_hdr_t         ibrm_hdr;             /* portals header */
 268         __u64             ibrm_cookie;          /* opaque completion cookie */
 269         __u32             ibrm_num_descs;       /* how many descs */
 270         kib_rdma_desc_t   ibrm_desc[0];         /* where to suck/blow */
 271 } kib_rdma_msg_t __attribute__((packed));
 272
 273 #define kib_rdma_msg_len(num_descs) \
 274         offsetof(kib_msg_t, ibm_u.rdma.ibrm_desc[num_descs])
 275
 276 typedef struct
 277 {
 278         __u64             ibcm_cookie;          /* opaque completion cookie */
 279         __u32             ibcm_status;          /* completion status */
 280 } kib_completion_msg_t __attribute__((packed));
 281
 282 typedef struct
 283 {
 284         __u32              ibm_magic;           /* I'm an openibnal message */
 285         __u16              ibm_version;         /* this is my version number */
 286         __u8               ibm_type;            /* msg type */
 287         __u8               ibm_credits;         /* returned credits */
 288 #if IBNAL_CKSUM
 289         __u32              ibm_nob;
 290         __u32              ibm_cksum;
 291 #endif
 292         union {
 293                 kib_immediate_msg_t   immediate;
 294                 kib_rdma_msg_t        rdma;
 295                 kib_completion_msg_t  completion;
 296         } ibm_u __attribute__((packed));
 297 } kib_msg_t __attribute__((packed));
 298
 299 #define IBNAL_MSG_MAGIC       0x0be91b91        /* unique magic */
 300 #define IBNAL_MSG_VERSION              1        /* current protocol version */
 301
 302 #define IBNAL_MSG_NOOP              0xd0        /* nothing (just credits) */
 303 #define IBNAL_MSG_IMMEDIATE         0xd1        /* portals hdr + payload */
 304 #define IBNAL_MSG_PUT_RDMA          0xd2        /* portals PUT hdr + source rdma desc */
 305 #define IBNAL_MSG_PUT_DONE          0xd3        /* signal PUT rdma completion */
 306 #define IBNAL_MSG_GET_RDMA          0xd4        /* portals GET hdr + sink rdma desc */
 307 #define IBNAL_MSG_GET_DONE          0xd5        /* signal GET rdma completion */
 308
 309 /***********************************************************************/
 310
 311 typedef struct kib_rx                           /* receive message */
 312 {
 313         struct list_head          rx_list;      /* queue for attention */
 314         struct kib_conn          *rx_conn;      /* owning conn */
 315         int                       rx_rdma;      /* RDMA completion posted? */
 316         int                       rx_posted;    /* posted? */
 317         __u64                     rx_vaddr;     /* pre-mapped buffer (hca vaddr) */
 318         kib_msg_t                *rx_msg;       /* pre-mapped buffer (host vaddr) */
 319         IB_WORK_REQ               rx_wrq;
 320         IB_LOCAL_DATASEGMENT      rx_gl;        /* and it's memory */
 321 } kib_rx_t;
 322
 323 typedef struct kib_tx                           /* transmit message */
 324 {
 325         struct list_head          tx_list;      /* queue on idle_txs ibc_tx_queue etc. */
 326         int                       tx_isnblk;    /* I'm reserved for non-blocking sends */
 327         struct kib_conn          *tx_conn;      /* owning conn */
 328         int                       tx_mapped;    /* mapped for RDMA? */
 329         int                       tx_sending;   /* # tx callbacks outstanding */
 330         int                       tx_status;    /* completion status */
 331         unsigned long             tx_deadline;  /* completion deadline */
 332         int                       tx_passive_rdma; /* peer sucks/blows */
 333         int                       tx_passive_rdma_wait; /* waiting for peer to complete */
 334         __u64                     tx_passive_rdma_cookie; /* completion cookie */
 335         lib_msg_t                *tx_libmsg[2]; /* lib msgs to finalize on completion */
 336         kib_md_t                  tx_md;        /* RDMA mapping (active/passive) */
 337         __u64                     tx_vaddr;     /* pre-mapped buffer (hca vaddr) */
 338         kib_msg_t                *tx_msg;       /* pre-mapped buffer (host vaddr) */
 339         int                       tx_nsp;       /* # send work items */
 340         IB_WORK_REQ               tx_wrq[IBNAL_TX_MAX_SG];    /* send work items... */
 341         IB_LOCAL_DATASEGMENT      tx_gl[IBNAL_TX_MAX_SG];     /* ...and their memory */
 342 } kib_tx_t;
 343
 344 #define KIB_TX_UNMAPPED       0
 345 #define KIB_TX_MAPPED         1
 346 #define KIB_TX_MAPPED_FMR     2
 347
 348 typedef struct kib_wire_connreq
 349 {
 350         __u32        wcr_magic;                 /* I'm an openibnal connreq */
 351         __u16        wcr_version;               /* this is my version number */
 352         __u16        wcr_queue_depth;           /* this is my receive queue size */
 353         __u64        wcr_nid;                   /* peer's NID */
 354         __u64        wcr_incarnation;           /* peer's incarnation */
 355 } kib_wire_connreq_t;
 356
 357 typedef struct kib_gid
 358 {
 359         __u64   hi, lo;
 360 } kib_gid_t;
 361
 362 typedef struct kib_connreq
 363 {
 364         /* connection-in-progress */
 365         struct kib_conn                    *cr_conn;
 366         kib_wire_connreq_t                  cr_wcr;
 367         __u64                               cr_tid;
 368         IB_SERVICE_RECORD                   cr_service;
 369         kib_gid_t                           cr_gid;
 370         IB_PATH_RECORD                      cr_path;
 371         CM_REQUEST_INFO                     cr_cmreq;
 372         CM_CONN_INFO                        cr_discarded;
 373         CM_REJECT_INFO                      cr_rej_info;
 374 } kib_connreq_t;
 375
 376 typedef struct kib_conn
 377 {
 378         struct kib_peer    *ibc_peer;           /* owning peer */
 379         struct list_head    ibc_list;           /* stash on peer's conn list */
 380         __u64               ibc_incarnation;    /* which instance of the peer */
 381         atomic_t            ibc_refcount;       /* # users */
 382         int                 ibc_state;          /* what's happening */
 383         atomic_t            ibc_nob;            /* # bytes buffered */
 384         int                 ibc_nsends_posted;  /* # uncompleted sends */
 385         int                 ibc_credits;        /* # credits I have */
 386         int                 ibc_outstanding_credits; /* # credits to return */
 387         int                 ibc_rcvd_disconnect;/* received discon request */
 388         int                 ibc_sent_disconnect;/* sent discon request */
 389         struct list_head    ibc_tx_queue;       /* send queue */
 390         struct list_head    ibc_active_txs;     /* active tx awaiting completion */
 391         spinlock_t          ibc_lock;           /* serialise */
 392         kib_rx_t           *ibc_rxs;            /* the rx descs */
 393         kib_pages_t        *ibc_rx_pages;       /* premapped rx msg pages */
 394         IB_HANDLE           ibc_qp;             /* queue pair */
 395         IB_HANDLE           ibc_cep;            /* connection ID? */
 396         IB_QP_ATTRIBUTES_QUERY ibc_qp_attrs;    /* QP attrs */
 397         kib_connreq_t      *ibc_connreq;        /* connection request state */
 398 } kib_conn_t;
 399
 400 #define IBNAL_CONN_INIT_NOTHING      0          /* initial state */
 401 #define IBNAL_CONN_INIT_QP           1          /* ibc_qp set up */
 402 #define IBNAL_CONN_CONNECTING        2          /* started to connect */
 403 #define IBNAL_CONN_ESTABLISHED       3          /* connection established */
 404 #define IBNAL_CONN_SEND_DREQ         4          /* to send disconnect req */
 405 #define IBNAL_CONN_DREQ              5          /* sent disconnect req */
 406 #define IBNAL_CONN_DREP              6          /* sent disconnect rep */
 407 #define IBNAL_CONN_DISCONNECTED      7          /* no more QP or CM traffic */
 408
 409 #define KIB_ASSERT_CONN_STATE(conn, state) do {                         \
 410         LASSERTF((conn)->ibc_state == state, "%d\n", conn->ibc_state);  \
 411 } while (0)
 412
 413 #define KIB_ASSERT_CONN_STATE_RANGE(conn, low, high) do {               \
 414         LASSERTF(low <= high, "%d %d\n", low, high);                    \
 415         LASSERTF((conn)->ibc_state >= low && (conn)->ibc_state <= high, \
 416                  "%d\n", conn->ibc_state);                              \
 417 } while (0)
 418
 419 typedef struct kib_peer
 420 {
 421         struct list_head    ibp_list;           /* stash on global peer list */
 422         struct list_head    ibp_connd_list;     /* schedule on kib_connd_peers */
 423         ptl_nid_t           ibp_nid;            /* who's on the other end(s) */
 424         atomic_t            ibp_refcount;       /* # users */
 425         int                 ibp_persistence;    /* "known" peer refs */
 426         struct list_head    ibp_conns;          /* all active connections */
 427         struct list_head    ibp_tx_queue;       /* msgs waiting for a conn */
 428         int                 ibp_connecting;     /* connecting+accepting */
 429         unsigned long       ibp_reconnect_time; /* when reconnect may be attempted */
 430         unsigned long       ibp_reconnect_interval; /* exponential backoff */
 431 } kib_peer_t;
 432
 433
 434 extern lib_nal_t       kibnal_lib;
 435 extern kib_data_t      kibnal_data;
 436 extern kib_tunables_t  kibnal_tunables;
 437
 438 /******************************************************************************/
 439 /* Infinicon IBT interface wrappers */
 440 #define IIBT_IF (kibnal_data.kib_interfaces.ver2)
 441
 442 static inline FSTATUS
 443 iibt_get_hca_guids(uint32 *hca_count, EUI64 *hca_guid_list)
 444 {
 445         return IIBT_IF.GetCaGuids(hca_count, hca_guid_list);
 446 }
 447
 448 static inline FSTATUS
 449 iibt_open_hca(EUI64                    hca_guid,
 450              IB_COMPLETION_CALLBACK   completion_callback,
 451              IB_ASYNC_EVENT_CALLBACK  async_event_callback,
 452              void                    *arg,
 453              IB_HANDLE               *handle)
 454 {
 455         return IIBT_IF.Vpi.OpenCA(hca_guid, completion_callback,
 456                                   async_event_callback, arg, handle);
 457 }
 458
 459 static inline FSTATUS
 460 iibt_query_hca(IB_HANDLE hca_handle, IB_CA_ATTRIBUTES *attrs, void **argp)
 461 {
 462         return IIBT_IF.Vpi.QueryCA(hca_handle, attrs, argp);
 463 }
 464
 465 static inline FSTATUS
 466 iibt_close_hca(IB_HANDLE hca_handle)
 467 {
 468         return IIBT_IF.Vpi.CloseCA(hca_handle);
 469 }
 470
 471 static inline FSTATUS
 472 iibt_pd_allocate(IB_HANDLE hca_handle, __u32 max_avs, IB_HANDLE *pd_handle)
 473 {
 474         return IIBT_IF.Vpi.AllocatePD(hca_handle, max_avs, pd_handle);
 475 }
 476
 477 static inline FSTATUS
 478 iibt_pd_free(IB_HANDLE pd_handle)
 479 {
 480         return IIBT_IF.Vpi.FreePD(pd_handle);
 481 }
 482
 483 static inline FSTATUS
 484 iibt_register_physical_memory(IB_HANDLE hca_handle,
 485                               IB_VIRT_ADDR requested_io_va,
 486                               void *phys_buffers, uint64 nphys_buffers,
 487                               uint32 io_va_offset, IB_HANDLE pd_handle,
 488                               IB_ACCESS_CONTROL access,
 489                               IB_HANDLE *mem_handle,
 490                               IB_VIRT_ADDR *actual_io_va,
 491                               IB_L_KEY *lkey, IB_R_KEY *rkey)
 492 {
 493         return IIBT_IF.Vpi.RegisterPhysMemRegion(hca_handle, requested_io_va,
 494                                                  phys_buffers, nphys_buffers,
 495                                                  io_va_offset, pd_handle,
 496                                                  access,
 497                                                  mem_handle, actual_io_va,
 498                                                  lkey, rkey);
 499 }
 500
 501 static inline FSTATUS
 502 iibt_register_contig_physical_memory(IB_HANDLE hca_handle,
 503                                      IB_VIRT_ADDR requested_io_va,
 504                                      IB_MR_PHYS_BUFFER *phys_buffers,
 505                                      uint64 nphys_buffers,
 506                                      uint32 io_va_offset, IB_HANDLE pd_handle,
 507                                      IB_ACCESS_CONTROL access,
 508                                      IB_HANDLE *mem_handle,
 509                                      IB_VIRT_ADDR *actual_io_va,
 510                                      IB_L_KEY *lkey, IB_R_KEY *rkey)
 511 {
 512         return IIBT_IF.Vpi.RegisterContigPhysMemRegion(hca_handle,
 513                                                        requested_io_va,
 514                                                        phys_buffers,
 515                                                        nphys_buffers,
 516                                                        io_va_offset, pd_handle,
 517                                                        access,
 518                                                        mem_handle, actual_io_va,
 519                                                        lkey, rkey);
 520 }
 521
 522 static inline FSTATUS
 523 iibt_register_memory(IB_HANDLE hca_handle,
 524                      void *virt_addr, unsigned int length,
 525                      IB_HANDLE pd_handle,
 526                      IB_ACCESS_CONTROL access,
 527                      IB_HANDLE *mem_handle,
 528                      IB_L_KEY *lkey, IB_R_KEY *rkey)
 529 {
 530         return IIBT_IF.Vpi.RegisterMemRegion(hca_handle,
 531                                              virt_addr, length,
 532                                              pd_handle,
 533                                              access,
 534                                              mem_handle,
 535                                              lkey, rkey);
 536 }
 537
 538 static inline FSTATUS
 539 iibt_deregister_memory(IB_HANDLE mem_handle)
 540 {
 541         return IIBT_IF.Vpi.DeregisterMemRegion(mem_handle);
 542 }
 543
 544 static inline FSTATUS
 545 iibt_cq_create(IB_HANDLE hca_handle, uint32 requested_size,
 546               void *arg, IB_HANDLE *cq_handle, uint32 *actual_size)
 547 {
 548         return IIBT_IF.Vpi.CreateCQ(hca_handle, requested_size,
 549                                    arg, cq_handle, actual_size);
 550 }
 551
 552 static inline FSTATUS
 553 iibt_cq_poll(IB_HANDLE cq_handle, IB_WORK_COMPLETION *wc)
 554 {
 555         return IIBT_IF.Vpi.PollCQ(cq_handle, wc);
 556 }
 557
 558 static inline FSTATUS
 559 iibt_cq_rearm(IB_HANDLE cq_handle, IB_CQ_EVENT_SELECT select)
 560 {
 561         return IIBT_IF.Vpi.RearmCQ(cq_handle, select);
 562 }
 563
 564 static inline FSTATUS
 565 iibt_cq_destroy(IB_HANDLE cq_handle)
 566 {
 567         return IIBT_IF.Vpi.DestroyCQ(cq_handle);
 568 }
 569
 570 static inline FSTATUS
 571 iibt_qp_create(IB_HANDLE hca_handle, IB_QP_ATTRIBUTES_CREATE *create_attr,
 572               void *arg, IB_HANDLE *cq_handle,
 573               IB_QP_ATTRIBUTES_QUERY *query_attr)
 574 {
 575         return IIBT_IF.Vpi.CreateQP(hca_handle, create_attr, arg, cq_handle,
 576                                     query_attr);
 577 }
 578
 579 static inline FSTATUS
 580 iibt_qp_query(IB_HANDLE qp_handle, IB_QP_ATTRIBUTES_QUERY *query_attr,
 581               void **arg_ptr)
 582 {
 583         return IIBT_IF.Vpi.QueryQP(qp_handle, query_attr, arg_ptr);
 584 }
 585
 586 static inline FSTATUS
 587 iibt_qp_modify(IB_HANDLE qp_handle, IB_QP_ATTRIBUTES_MODIFY *modify_attr,
 588                IB_QP_ATTRIBUTES_QUERY *query_attr)
 589 {
 590         return IIBT_IF.Vpi.ModifyQP(qp_handle, modify_attr, query_attr);
 591 }
 592
 593 static inline FSTATUS
 594 iibt_qp_destroy(IB_HANDLE qp_handle)
 595 {
 596         return IIBT_IF.Vpi.DestroyQP(qp_handle);
 597 }
 598
 599 static inline FSTATUS
 600 iibt_postrecv(IB_HANDLE qp_handle, IB_WORK_REQ *work_req)
 601 {
 602         return IIBT_IF.Vpi.PostRecv(qp_handle, work_req);
 603 }
 604
 605 static inline FSTATUS
 606 iibt_postsend(IB_HANDLE qp_handle, IB_WORK_REQ *work_req)
 607 {
 608         return IIBT_IF.Vpi.PostSend(qp_handle, work_req);
 609 }
 610
 611 static inline FSTATUS
 612 iibt_sd_register(IB_HANDLE *sd_handle, CLIENT_CONTROL_PARAMETERS *p)
 613 {
 614         return IIBT_IF.Sdi.Register(sd_handle, p);
 615 }
 616
 617 static inline FSTATUS
 618 iibt_sd_deregister(IB_HANDLE sd_handle)
 619 {
 620         return IIBT_IF.Sdi.Deregister(sd_handle);
 621 }
 622
 623 static inline FSTATUS
 624 iibt_sd_port_fabric_operation(IB_HANDLE sd_handle, EUI64 port_guid,
 625                               FABRIC_OPERATION_DATA *fod,
 626                               PFABRIC_OPERATION_CALLBACK callback,
 627                               COMMAND_CONTROL_PARAMETERS *p, void *arg)
 628 {
 629         return IIBT_IF.Sdi.PortFabricOperation(sd_handle, port_guid,
 630                                                fod, callback, p, arg);
 631 }
 632
 633 static inline FSTATUS
 634 iibt_sd_query_port_fabric_information(IB_HANDLE sd_handle, EUI64 port_guid,
 635                                       QUERY *qry,
 636                                       PQUERY_CALLBACK callback,
 637                                       COMMAND_CONTROL_PARAMETERS *p, void *arg)
 638 {
 639         return IIBT_IF.Sdi.QueryPortFabricInformation(sd_handle, port_guid,
 640                                                       qry, callback, p, arg);
 641 }
 642
 643 static inline IB_HANDLE
 644 iibt_cm_create_cep(CM_CEP_TYPE type)
 645 {
 646         return IIBT_IF.Cmi.CmCreateCEP(type);
 647 }
 648
 649 static inline FSTATUS
 650 iibt_cm_modify_cep(IB_HANDLE cep, uint32 attr, char* value, uint32 len,
 651                    uint32 offset)
 652 {
 653         return IIBT_IF.Cmi.CmModifyCEP(cep, attr, value, len, offset);
 654 }
 655
 656 static inline FSTATUS
 657 iibt_cm_destroy_cep(IB_HANDLE cep_handle)
 658 {
 659         return IIBT_IF.Cmi.CmDestroyCEP(cep_handle);
 660 }
 661
 662 static inline FSTATUS
 663 iibt_cm_listen(IB_HANDLE cep, CM_LISTEN_INFO *info,
 664                PFN_CM_CALLBACK callback, void *arg)
 665 {
 666         return IIBT_IF.Cmi.CmListen(cep, info, callback, arg);
 667 }
 668
 669 static inline FSTATUS
 670 iibt_cm_cancel(IB_HANDLE cep)
 671 {
 672         return IIBT_IF.Cmi.CmCancel(cep);
 673 }
 674
 675 static inline FSTATUS
 676 iibt_cm_accept(IB_HANDLE cep,
 677                CM_CONN_INFO *send_info, CM_CONN_INFO *recv_info,
 678                PFN_CM_CALLBACK callback, void *arg,
 679                IB_HANDLE *new_cep)
 680 {
 681         return IIBT_IF.Cmi.CmAccept(cep,
 682                                     send_info, recv_info,
 683                                     callback, arg, new_cep);
 684 }
 685
 686 static inline FSTATUS
 687 iibt_cm_reject(IB_HANDLE cep, CM_REJECT_INFO *rej)
 688 {
 689         return IIBT_IF.Cmi.CmReject(cep, rej);
 690 }
 691
 692 static inline FSTATUS
 693 iibt_cm_disconnect(IB_HANDLE cep, CM_DREQUEST_INFO *req,
 694                    CM_DREPLY_INFO *reply)
 695 {
 696         return IIBT_IF.Cmi.CmDisconnect(cep, req, reply);
 697 }
 698
 699 static inline FSTATUS
 700 iibt_cm_connect (IB_HANDLE cep, CM_REQUEST_INFO *req,
 701                  PFN_CM_CALLBACK callback, void *arg)
 702 {
 703         return IIBT_IF.Cmi.CmConnect (cep, req, callback, arg);
 704 }
 705
 706 static inline int wrq_signals_completion(IB_WORK_REQ *wrq)
 707 {
 708         return wrq->Req.SendRC.Options.s.SignaledCompletion == 1;
 709 }
 710
 711
 712 /******************************************************************************/
 713
 714 /* these are purposely avoiding using local vars so they don't increase
 715  * stack consumption. */
 716
 717 #define kib_peer_addref(peer) do {                                      \
 718         LASSERTF(atomic_read(&peer->ibp_refcount) > 0, "%d\n",          \
 719                  atomic_read(&peer->ibp_refcount));                     \
 720         CDEBUG(D_NET, "++peer[%p] -> "LPX64" (%d)\n",                   \
 721                peer, peer->ibp_nid, atomic_read (&peer->ibp_refcount)); \
 722         atomic_inc(&peer->ibp_refcount);                                \
 723 } while (0)
 724
 725 #define kib_peer_decref(peer) do {                                      \
 726         LASSERTF(atomic_read(&peer->ibp_refcount) > 0, "%d\n",          \
 727                  atomic_read(&peer->ibp_refcount));                     \
 728         CDEBUG(D_NET, "--peer[%p] -> "LPX64" (%d)\n",                   \
 729                peer, peer->ibp_nid, atomic_read (&peer->ibp_refcount)); \
 730         if (atomic_dec_and_test (&peer->ibp_refcount)) {                \
 731                 CDEBUG (D_NET, "destroying peer "LPX64" %p\n",          \
 732                         peer->ibp_nid, peer);                           \
 733                 kibnal_destroy_peer (peer);                             \
 734         }                                                               \
 735 } while (0)
 736
 737 /******************************************************************************/
 738
 739 static inline struct list_head *
 740 kibnal_nid2peerlist (ptl_nid_t nid)
 741 {
 742         unsigned int hash = ((unsigned int)nid) % kibnal_data.kib_peer_hash_size;
 743
 744         return (&kibnal_data.kib_peers [hash]);
 745 }
 746
 747 static inline int
 748 kibnal_peer_active(kib_peer_t *peer)
 749 {
 750         /* Am I in the peer hash table? */
 751         return (!list_empty(&peer->ibp_list));
 752 }
 753
 754 static inline void
 755 kibnal_queue_tx_locked (kib_tx_t *tx, kib_conn_t *conn)
 756 {
 757         /* CAVEAT EMPTOR: tx takes caller's ref on conn */
 758
 759         LASSERT (tx->tx_nsp > 0);               /* work items set up */
 760         LASSERT (tx->tx_conn == NULL);          /* only set here */
 761
 762         tx->tx_conn = conn;
 763         tx->tx_deadline = jiffies + kibnal_tunables.kib_io_timeout * HZ;
 764         list_add_tail(&tx->tx_list, &conn->ibc_tx_queue);
 765 }
 766
 767 #define KIBNAL_SERVICE_KEY_MASK  (IB_SERVICE_RECORD_COMP_SERVICENAME |          \
 768                                   IB_SERVICE_RECORD_COMP_SERVICEDATA8_1 |       \
 769                                   IB_SERVICE_RECORD_COMP_SERVICEDATA8_2 |       \
 770                                   IB_SERVICE_RECORD_COMP_SERVICEDATA8_3 |       \
 771                                   IB_SERVICE_RECORD_COMP_SERVICEDATA8_4 |       \
 772                                   IB_SERVICE_RECORD_COMP_SERVICEDATA8_5 |       \
 773                                   IB_SERVICE_RECORD_COMP_SERVICEDATA8_6 |       \
 774                                   IB_SERVICE_RECORD_COMP_SERVICEDATA8_7 |       \
 775                                   IB_SERVICE_RECORD_COMP_SERVICEDATA8_8)
 776
 777 static inline __u64*
 778 kibnal_service_nid_field(IB_SERVICE_RECORD *srv)
 779 {
 780         /* must be consistent with KIBNAL_SERVICE_KEY_MASK */
 781         return (__u64 *)srv->ServiceData8;
 782 }
 783
 784
 785 static inline void
 786 kibnal_set_service_keys(IB_SERVICE_RECORD *srv, ptl_nid_t nid)
 787 {
 788         LASSERT (strlen(IBNAL_SERVICE_NAME) < sizeof(srv->ServiceName));
 789         memset (srv->ServiceName, 0, sizeof(srv->ServiceName));
 790         strcpy (srv->ServiceName, IBNAL_SERVICE_NAME);
 791
 792         *kibnal_service_nid_field(srv) = cpu_to_le64(nid);
 793 }
 794
 795 #if 0
 796 static inline void
 797 kibnal_show_rdma_attr (kib_conn_t *conn)
 798 {
 799         struct ib_qp_attribute qp_attr;
 800         int                    rc;
 801
 802         memset (&qp_attr, 0, sizeof(qp_attr));
 803         rc = ib_qp_query(conn->ibc_qp, &qp_attr);
 804         if (rc != 0) {
 805                 CERROR ("Can't get qp attrs: %d\n", rc);
 806                 return;
 807         }
 808
 809         CWARN ("RDMA CAPABILITY: write %s read %s\n",
 810                (qp_attr.valid_fields & TS_IB_QP_ATTRIBUTE_RDMA_ATOMIC_ENABLE) ?
 811                (qp_attr.enable_rdma_write ? "enabled" : "disabled") : "invalid",
 812                (qp_attr.valid_fields & TS_IB_QP_ATTRIBUTE_RDMA_ATOMIC_ENABLE) ?
 813                (qp_attr.enable_rdma_read ? "enabled" : "disabled") : "invalid");
 814 }
 815 #endif
 816
 817 #if CONFIG_X86
 818 static inline __u64
 819 kibnal_page2phys (struct page *p)
 820 {
 821         __u64 page_number = p - mem_map;
 822
 823         return (page_number << PAGE_SHIFT);
 824 }
 825 #else
 826 # error "no page->phys"
 827 #endif
 828
 829 /* CAVEAT EMPTOR:
 830  * We rely on tx/rx descriptor alignment to allow us to use the lowest bit
 831  * of the work request id as a flag to determine if the completion is for a
 832  * transmit or a receive.  It seems that that the CQ entry's 'op' field
 833  * isn't always set correctly on completions that occur after QP teardown. */
 834
 835 static inline __u64
 836 kibnal_ptr2wreqid (void *ptr, int isrx)
 837 {
 838         unsigned long lptr = (unsigned long)ptr;
 839
 840         LASSERT ((lptr & 1) == 0);
 841         return (__u64)(lptr | (isrx ? 1 : 0));
 842 }
 843
 844 static inline void *
 845 kibnal_wreqid2ptr (__u64 wreqid)
 846 {
 847         return (void *)(((unsigned long)wreqid) & ~1UL);
 848 }
 849
 850 static inline int
 851 kibnal_wreqid_is_rx (__u64 wreqid)
 852 {
 853         return (wreqid & 1) != 0;
 854 }
 855
 856 static inline int
 857 kibnal_whole_mem(void)
 858 {
 859         return kibnal_data.kib_md.md_handle != NULL;
 860 }
 861
 862 extern kib_peer_t *kibnal_create_peer (ptl_nid_t nid);
 863 extern void kibnal_destroy_peer (kib_peer_t *peer);
 864 extern int kibnal_del_peer (ptl_nid_t nid, int single_share);
 865 extern kib_peer_t *kibnal_find_peer_locked (ptl_nid_t nid);
 866 extern void kibnal_unlink_peer_locked (kib_peer_t *peer);
 867 extern int  kibnal_close_stale_conns_locked (kib_peer_t *peer,
 868                                               __u64 incarnation);
 869 extern kib_conn_t *kibnal_create_conn (void);
 870 extern void kibnal_put_conn (kib_conn_t *conn);
 871 extern void kibnal_destroy_conn (kib_conn_t *conn);
 872 void kibnal_listen_callback(IB_HANDLE cep, CM_CONN_INFO *info, void *arg);
 873
 874 extern int kibnal_alloc_pages (kib_pages_t **pp, int npages, int access);
 875 extern void kibnal_free_pages (kib_pages_t *p);
 876
 877 extern void kibnal_check_sends (kib_conn_t *conn);
 878 extern void kibnal_close_conn_locked (kib_conn_t *conn, int error);
 879 extern void kibnal_destroy_conn (kib_conn_t *conn);
 880 extern int  kibnal_thread_start (int (*fn)(void *arg), void *arg);
 881 extern int  kibnal_scheduler(void *arg);
 882 extern int  kibnal_connd (void *arg);
 883 extern void kibnal_init_tx_msg (kib_tx_t *tx, int type, int body_nob);
 884 extern void kibnal_close_conn (kib_conn_t *conn, int why);
 885 extern void kibnal_start_active_rdma (int type, int status,
 886                                       kib_rx_t *rx, lib_msg_t *libmsg,
 887                                       unsigned int niov,
 888                                       struct iovec *iov, ptl_kiov_t *kiov,
 889                                       size_t offset, size_t nob);
 890
 891 void kibnal_ca_async_callback (void *ca_arg, IB_EVENT_RECORD *ev);
 892 void kibnal_ca_callback (void *ca_arg, void *cq_arg);