1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Copyright (C) 2004 Cluster File Systems, Inc.
5 * Author: Eric Barton <eric@bartonsoftware.com>
7 * This file is part of Lustre, http://www.lustre.org.
9 * Lustre is free software; you can redistribute it and/or
10 * modify it under the terms of version 2 of the GNU General Public
11 * License as published by the Free Software Foundation.
13 * Lustre is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with Lustre; if not, write to the Free Software
20 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
25 # define EXPORT_SYMTAB
28 #include <linux/config.h>
29 #include <linux/module.h>
30 #include <linux/kernel.h>
32 #include <linux/string.h>
33 #include <linux/stat.h>
34 #include <linux/errno.h>
35 #include <linux/smp_lock.h>
36 #include <linux/unistd.h>
37 #include <linux/uio.h>
39 #include <asm/system.h>
40 #include <asm/uaccess.h>
43 #include <linux/init.h>
45 #include <linux/file.h>
46 #include <linux/stat.h>
47 #include <linux/list.h>
48 #include <linux/kmod.h>
49 #include <linux/sysctl.h>
51 #define DEBUG_SUBSYSTEM S_NAL
53 #include <linux/kp30.h>
54 #include <portals/p30.h>
55 #include <portals/lib-p30.h>
56 #include <portals/nal.h>
58 #include <linux/iba/ibt.h>
60 #define GCC_VERSION (__GNUC__ * 10000 \
61 + __GNUC_MINOR__ * 100 \
62 + __GNUC_PATCHLEVEL__)
64 /* Test for GCC > 3.2.2 */
65 #if GCC_VERSION <= 30202
66 /* GCC 3.2.2, and presumably several versions before it, will
67 * miscompile this driver. See
68 * http://gcc.gnu.org/bugzilla/show_bug.cgi?id=9853. */
69 #error Invalid GCC version. Must use GCC >= 3.2.3
72 #define IBNAL_SERVICE_NAME "iibnal"
73 #define IBNAL_SERVICE_NUMBER 0x11b9a1
76 # define IBNAL_N_SCHED num_online_cpus() /* # schedulers */
78 # define IBNAL_N_SCHED 1 /* # schedulers */
81 #define IBNAL_MIN_RECONNECT_INTERVAL HZ /* first failed connection retry... */
82 #define IBNAL_MAX_RECONNECT_INTERVAL (60*HZ) /* ...exponentially increasing to this */
84 #define IBNAL_MSG_SIZE (4<<10) /* max size of queued messages (inc hdr) */
86 #define IBNAL_MSG_QUEUE_SIZE 8 /* # messages/RDMAs in-flight */
87 #define IBNAL_CREDIT_HIGHWATER 7 /* when to eagerly return credits */
88 /* 7 indicates infinite retry attempts, Infinicon recommended 5 */
89 #define IBNAL_RETRY 5 /* # times to retry */
90 #define IBNAL_RNR_RETRY 5 /* */
91 #define IBNAL_CM_RETRY 5 /* # times to retry connection */
92 #define IBNAL_FLOW_CONTROL 1
93 #define IBNAL_ACK_TIMEOUT 20 /* supposedly 4 secs */
95 #define IBNAL_NTX 64 /* # tx descs */
96 /* this had to be dropped down so that we only register < 255 pages per
97 * region. this will change if we register all memory. */
98 #define IBNAL_NTX_NBLK 128 /* # reserved tx descs */
100 #define IBNAL_PEER_HASH_SIZE 101 /* # peer lists */
102 #define IBNAL_RESCHED 100 /* # scheduler loops before reschedule */
104 #define IBNAL_CONCURRENT_PEERS 1000 /* # nodes all talking at once to me */
106 /* default vals for runtime tunables */
107 #define IBNAL_IO_TIMEOUT 50 /* default comms timeout (seconds) */
109 /************************/
110 /* derived constants... */
112 /* TX messages (shared by all connections) */
113 #define IBNAL_TX_MSGS (IBNAL_NTX + IBNAL_NTX_NBLK)
114 #define IBNAL_TX_MSG_BYTES (IBNAL_TX_MSGS * IBNAL_MSG_SIZE)
115 #define IBNAL_TX_MSG_PAGES ((IBNAL_TX_MSG_BYTES + PAGE_SIZE - 1)/PAGE_SIZE)
117 #define IBNAL_TX_MAX_SG (PTL_MD_MAX_IOV + 1)
119 /* RX messages (per connection) */
120 #define IBNAL_RX_MSGS IBNAL_MSG_QUEUE_SIZE
121 #define IBNAL_RX_MSG_BYTES (IBNAL_RX_MSGS * IBNAL_MSG_SIZE)
122 #define IBNAL_RX_MSG_PAGES ((IBNAL_RX_MSG_BYTES + PAGE_SIZE - 1)/PAGE_SIZE)
125 /* we may have up to 2 completions per transmit +
126 1 completion per receive, per connection */
127 #define IBNAL_CQ_ENTRIES ((2*IBNAL_TX_MSGS) + \
128 (IBNAL_RX_MSGS * IBNAL_CONCURRENT_PEERS))
130 #define IBNAL_RDMA_BASE 0x0eeb0000
132 #define IBNAL_WHOLE_MEM 1
133 #define IBNAL_CKSUM 0
134 //#define IBNAL_CALLBACK_CTXT IB_CQ_CALLBACK_PROCESS
135 #define IBNAL_CALLBACK_CTXT IB_CQ_CALLBACK_INTERRUPT
137 /* XXX I have no idea. */
138 #define IBNAL_STARTING_PSN 1
142 int kib_io_timeout; /* comms timeout (seconds) */
143 struct ctl_table_header *kib_sysctl; /* sysctl interface */
146 /* some of these have specific types in the stack that just map back
147 * to the uFOO types, like IB_{L,R}_KEY. */
150 int ibp_npages; /* # pages */
151 int ibp_mapped; /* mapped? */
152 __u64 ibp_vaddr; /* mapped region vaddr */
153 __u32 ibp_lkey; /* mapped region lkey */
154 __u32 ibp_rkey; /* mapped region rkey */
155 IB_HANDLE ibp_handle; /* mapped region handle */
156 struct page *ibp_pages[0];
165 } kib_md_t __attribute__((packed));
169 int kib_init; /* initialisation state */
170 __u64 kib_incarnation; /* which one am I */
171 int kib_shutdown; /* shut down? */
172 atomic_t kib_nthreads; /* # live threads */
174 __u64 kib_service_id; /* service number I listen on */
175 __u64 kib_port_guid; /* my GUID (lo 64 of GID)*/
176 __u16 kib_port_pkey; /* my pkey, whatever that is */
177 ptl_nid_t kib_nid; /* my NID */
178 struct semaphore kib_nid_mutex; /* serialise NID ops */
179 struct semaphore kib_nid_signal; /* signal completion */
180 IB_HANDLE kib_cep; /* connection end point */
182 rwlock_t kib_global_lock; /* stabilize peer/conn ops */
184 struct list_head *kib_peers; /* hash table of all my known peers */
185 int kib_peer_hash_size; /* size of kib_peers */
186 atomic_t kib_npeers; /* # peers extant */
187 atomic_t kib_nconns; /* # connections extant */
189 struct list_head kib_connd_conns; /* connections to progress */
190 struct list_head kib_connd_peers; /* peers waiting for a connection */
191 wait_queue_head_t kib_connd_waitq; /* connection daemons sleep here */
192 unsigned long kib_connd_waketime; /* when connd will wake */
193 spinlock_t kib_connd_lock; /* serialise */
195 wait_queue_head_t kib_sched_waitq; /* schedulers sleep here */
196 struct list_head kib_sched_txq; /* tx requiring attention */
197 struct list_head kib_sched_rxq; /* rx requiring attention */
198 spinlock_t kib_sched_lock; /* serialise */
200 struct kib_tx *kib_tx_descs; /* all the tx descriptors */
201 kib_pages_t *kib_tx_pages; /* premapped tx msg pages */
203 struct list_head kib_idle_txs; /* idle tx descriptors */
204 struct list_head kib_idle_nblk_txs; /* idle reserved tx descriptors */
205 wait_queue_head_t kib_idle_tx_waitq; /* block here for tx descriptor */
206 __u64 kib_next_tx_cookie; /* RDMA completion cookie */
207 spinlock_t kib_tx_lock; /* serialise */
209 IB_HANDLE kib_hca; /* The HCA */
210 int kib_port; /* port on the device */
211 IB_HANDLE kib_pd; /* protection domain */
212 IB_HANDLE kib_sd; /* SD handle */
213 IB_HANDLE kib_cq; /* completion queue */
214 kib_md_t kib_md; /* full-mem registration */
216 void *kib_listen_handle; /* where I listen for connections */
218 IBT_INTERFACE_UNION kib_interfaces; /* The Infinicon IBT interface */
220 uint64 kib_hca_guids[8]; /* all the HCA guids */
221 IB_CA_ATTRIBUTES kib_hca_attrs; /* where to get HCA attrs */
222 FABRIC_OPERATION_DATA kib_fabopdata; /* (un)advertise service record */
225 #define IBNAL_INIT_NOTHING 0
226 #define IBNAL_INIT_DATA 1
227 #define IBNAL_INIT_LIB 2
228 #define IBNAL_INIT_HCA 3
229 #define IBNAL_INIT_PORTATTRS 4
230 #define IBNAL_INIT_PORT 5
231 #define IBNAL_INIT_SD 6
232 #define IBNAL_INIT_PD 7
233 #define IBNAL_INIT_FMR 8
234 #define IBNAL_INIT_MR 9
235 #define IBNAL_INIT_TXD 10
236 #define IBNAL_INIT_CQ 11
237 #define IBNAL_INIT_ALL 12
239 /************************************************************************
240 * Wire message structs.
241 * These are sent in sender's byte order (i.e. receiver flips).
242 * CAVEAT EMPTOR: other structs communicated between nodes (e.g. MAD
243 * private data and SM service info), is LE on the wire.
246 /* also kib_md_t above */
250 __u32 rd_nob; /* # of bytes */
251 __u64 rd_addr; /* remote io vaddr */
252 } kib_rdma_desc_t __attribute__((packed));
256 ptl_hdr_t ibim_hdr; /* portals header */
257 char ibim_payload[0]; /* piggy-backed payload */
258 } kib_immediate_msg_t __attribute__((packed));
260 /* these arrays serve two purposes during rdma. they are built on the passive
261 * side and sent to the active side as remote arguments. On the active side
262 * the descs are used as a data structure on the way to local gather items.
263 * the different roles result in split local/remote meaning of desc->rd_key */
266 ptl_hdr_t ibrm_hdr; /* portals header */
267 __u64 ibrm_cookie; /* opaque completion cookie */
268 __u32 ibrm_num_descs; /* how many descs */
269 __u32 rd_key; /* remote key */
270 kib_rdma_desc_t ibrm_desc[0]; /* where to suck/blow */
271 } kib_rdma_msg_t __attribute__((packed));
273 #define kib_rdma_msg_len(num_descs) \
274 offsetof(kib_msg_t, ibm_u.rdma.ibrm_desc[num_descs])
278 __u64 ibcm_cookie; /* opaque completion cookie */
279 __u32 ibcm_status; /* completion status */
280 } kib_completion_msg_t __attribute__((packed));
284 __u32 ibm_magic; /* I'm an openibnal message */
285 __u16 ibm_version; /* this is my version number */
286 __u8 ibm_type; /* msg type */
287 __u8 ibm_credits; /* returned credits */
293 kib_immediate_msg_t immediate;
295 kib_completion_msg_t completion;
296 } ibm_u __attribute__((packed));
297 } kib_msg_t __attribute__((packed));
299 #define IBNAL_MSG_MAGIC 0x0be91b91 /* unique magic */
300 #define IBNAL_MSG_VERSION 1 /* current protocol version */
302 #define IBNAL_MSG_NOOP 0xd0 /* nothing (just credits) */
303 #define IBNAL_MSG_IMMEDIATE 0xd1 /* portals hdr + payload */
304 #define IBNAL_MSG_PUT_RDMA 0xd2 /* portals PUT hdr + source rdma desc */
305 #define IBNAL_MSG_PUT_DONE 0xd3 /* signal PUT rdma completion */
306 #define IBNAL_MSG_GET_RDMA 0xd4 /* portals GET hdr + sink rdma desc */
307 #define IBNAL_MSG_GET_DONE 0xd5 /* signal GET rdma completion */
309 /***********************************************************************/
311 typedef struct kib_rx /* receive message */
313 struct list_head rx_list; /* queue for attention */
314 struct kib_conn *rx_conn; /* owning conn */
315 int rx_rdma; /* RDMA completion posted? */
316 int rx_posted; /* posted? */
317 __u64 rx_vaddr; /* pre-mapped buffer (hca vaddr) */
318 kib_msg_t *rx_msg; /* pre-mapped buffer (host vaddr) */
320 IB_LOCAL_DATASEGMENT rx_gl; /* and its memory */
323 typedef struct kib_tx /* transmit message */
325 struct list_head tx_list; /* queue on idle_txs ibc_tx_queue etc. */
326 int tx_isnblk; /* I'm reserved for non-blocking sends */
327 struct kib_conn *tx_conn; /* owning conn */
328 int tx_mapped; /* mapped for RDMA? */
329 int tx_sending; /* # tx callbacks outstanding */
330 int tx_status; /* completion status */
331 unsigned long tx_deadline; /* completion deadline */
332 int tx_passive_rdma; /* peer sucks/blows */
333 int tx_passive_rdma_wait; /* waiting for peer to complete */
334 __u64 tx_passive_rdma_cookie; /* completion cookie */
335 lib_msg_t *tx_libmsg[2]; /* lib msgs to finalize on completion */
336 kib_md_t tx_md; /* RDMA mapping (active/passive) */
337 __u64 tx_vaddr; /* pre-mapped buffer (hca vaddr) */
338 kib_msg_t *tx_msg; /* pre-mapped buffer (host vaddr) */
339 int tx_nsp; /* # send work items */
340 IB_WORK_REQ tx_wrq[IBNAL_TX_MAX_SG]; /* send work items... */
341 IB_LOCAL_DATASEGMENT tx_gl[IBNAL_TX_MAX_SG]; /* ...and their memory */
344 #define KIB_TX_UNMAPPED 0
345 #define KIB_TX_MAPPED 1
346 #define KIB_TX_MAPPED_FMR 2
348 typedef struct kib_wire_connreq
350 __u32 wcr_magic; /* I'm an openibnal connreq */
351 __u16 wcr_version; /* this is my version number */
352 __u16 wcr_queue_depth; /* this is my receive queue size */
353 __u64 wcr_nid; /* peer's NID */
354 __u64 wcr_incarnation; /* peer's incarnation */
355 } kib_wire_connreq_t;
357 typedef struct kib_gid
362 typedef struct kib_connreq
364 /* connection-in-progress */
365 struct kib_conn *cr_conn;
366 kib_wire_connreq_t cr_wcr;
368 IB_SERVICE_RECORD cr_service;
370 IB_PATH_RECORD cr_path;
371 CM_REQUEST_INFO cr_cmreq;
372 CM_CONN_INFO cr_discarded;
375 typedef struct kib_conn
377 struct kib_peer *ibc_peer; /* owning peer */
378 struct list_head ibc_list; /* stash on peer's conn list */
379 __u64 ibc_incarnation; /* which instance of the peer */
380 atomic_t ibc_refcount; /* # users */
381 int ibc_state; /* what's happening */
382 atomic_t ibc_nob; /* # bytes buffered */
383 int ibc_nsends_posted; /* # uncompleted sends */
384 int ibc_credits; /* # credits I have */
385 int ibc_outstanding_credits; /* # credits to return */
386 int ibc_rcvd_disconnect;/* received discon request */
387 int ibc_sent_disconnect;/* sent discon request */
388 struct list_head ibc_tx_queue; /* send queue */
389 struct list_head ibc_active_txs; /* active tx awaiting completion */
390 spinlock_t ibc_lock; /* serialise */
391 kib_rx_t *ibc_rxs; /* the rx descs */
392 kib_pages_t *ibc_rx_pages; /* premapped rx msg pages */
393 IB_HANDLE ibc_qp; /* queue pair */
394 IB_HANDLE ibc_cep; /* connection ID? */
395 IB_QP_ATTRIBUTES_QUERY ibc_qp_attrs; /* QP attrs */
396 kib_connreq_t *ibc_connreq; /* connection request state */
399 #define IBNAL_CONN_INIT_NOTHING 0 /* initial state */
400 #define IBNAL_CONN_INIT_QP 1 /* ibc_qp set up */
401 #define IBNAL_CONN_CONNECTING 2 /* started to connect */
402 #define IBNAL_CONN_ESTABLISHED 3 /* connection established */
403 #define IBNAL_CONN_SEND_DREQ 4 /* to send disconnect req */
404 #define IBNAL_CONN_DREQ 5 /* sent disconnect req */
405 #define IBNAL_CONN_DREP 6 /* sent disconnect rep */
406 #define IBNAL_CONN_DISCONNECTED 7 /* no more QP or CM traffic */
408 #define KIB_ASSERT_CONN_STATE(conn, state) do { \
409 LASSERTF((conn)->ibc_state == state, "%d\n", conn->ibc_state); \
412 #define KIB_ASSERT_CONN_STATE_RANGE(conn, low, high) do { \
413 LASSERTF(low <= high, "%d %d\n", low, high); \
414 LASSERTF((conn)->ibc_state >= low && (conn)->ibc_state <= high, \
415 "%d\n", conn->ibc_state); \
418 typedef struct kib_peer
420 struct list_head ibp_list; /* stash on global peer list */
421 struct list_head ibp_connd_list; /* schedule on kib_connd_peers */
422 ptl_nid_t ibp_nid; /* who's on the other end(s) */
423 atomic_t ibp_refcount; /* # users */
424 int ibp_persistence; /* "known" peer refs */
425 struct list_head ibp_conns; /* all active connections */
426 struct list_head ibp_tx_queue; /* msgs waiting for a conn */
427 int ibp_connecting; /* connecting+accepting */
428 unsigned long ibp_reconnect_time; /* when reconnect may be attempted */
429 unsigned long ibp_reconnect_interval; /* exponential backoff */
433 extern lib_nal_t kibnal_lib;
434 extern kib_data_t kibnal_data;
435 extern kib_tunables_t kibnal_tunables;
437 /******************************************************************************/
438 /* Infinicon IBT interface wrappers */
439 #define IIBT_IF (kibnal_data.kib_interfaces.ver2)
441 static inline FSTATUS
442 iibt_get_hca_guids(uint32 *hca_count, EUI64 *hca_guid_list)
444 return IIBT_IF.GetCaGuids(hca_count, hca_guid_list);
447 static inline FSTATUS
448 iibt_open_hca(EUI64 hca_guid,
449 IB_COMPLETION_CALLBACK completion_callback,
450 IB_ASYNC_EVENT_CALLBACK async_event_callback,
454 return IIBT_IF.Vpi.OpenCA(hca_guid, completion_callback,
455 async_event_callback, arg, handle);
458 static inline FSTATUS
459 iibt_query_hca(IB_HANDLE hca_handle, IB_CA_ATTRIBUTES *attrs, void **argp)
461 return IIBT_IF.Vpi.QueryCA(hca_handle, attrs, argp);
464 static inline FSTATUS
465 iibt_close_hca(IB_HANDLE hca_handle)
467 return IIBT_IF.Vpi.CloseCA(hca_handle);
470 static inline FSTATUS
471 iibt_pd_allocate(IB_HANDLE hca_handle, __u32 max_avs, IB_HANDLE *pd_handle)
473 return IIBT_IF.Vpi.AllocatePD(hca_handle, max_avs, pd_handle);
476 static inline FSTATUS
477 iibt_pd_free(IB_HANDLE pd_handle)
479 return IIBT_IF.Vpi.FreePD(pd_handle);
482 static inline FSTATUS
483 iibt_register_physical_memory(IB_HANDLE hca_handle,
484 IB_VIRT_ADDR requested_io_va,
485 void *phys_buffers, uint64 nphys_buffers,
486 uint32 io_va_offset, IB_HANDLE pd_handle,
487 IB_ACCESS_CONTROL access,
488 IB_HANDLE *mem_handle,
489 IB_VIRT_ADDR *actual_io_va,
490 IB_L_KEY *lkey, IB_R_KEY *rkey)
492 return IIBT_IF.Vpi.RegisterPhysMemRegion(hca_handle, requested_io_va,
493 phys_buffers, nphys_buffers,
494 io_va_offset, pd_handle,
496 mem_handle, actual_io_va,
500 static inline FSTATUS
501 iibt_register_contig_physical_memory(IB_HANDLE hca_handle,
502 IB_VIRT_ADDR requested_io_va,
503 IB_MR_PHYS_BUFFER *phys_buffers,
504 uint64 nphys_buffers,
505 uint32 io_va_offset, IB_HANDLE pd_handle,
506 IB_ACCESS_CONTROL access,
507 IB_HANDLE *mem_handle,
508 IB_VIRT_ADDR *actual_io_va,
509 IB_L_KEY *lkey, IB_R_KEY *rkey)
511 return IIBT_IF.Vpi.RegisterContigPhysMemRegion(hca_handle,
515 io_va_offset, pd_handle,
517 mem_handle, actual_io_va,
521 static inline FSTATUS
522 iibt_register_memory(IB_HANDLE hca_handle,
523 void *virt_addr, unsigned int length,
525 IB_ACCESS_CONTROL access,
526 IB_HANDLE *mem_handle,
527 IB_L_KEY *lkey, IB_R_KEY *rkey)
529 return IIBT_IF.Vpi.RegisterMemRegion(hca_handle,
537 static inline FSTATUS
538 iibt_deregister_memory(IB_HANDLE mem_handle)
540 return IIBT_IF.Vpi.DeregisterMemRegion(mem_handle);
543 static inline FSTATUS
544 iibt_cq_create(IB_HANDLE hca_handle, uint32 requested_size,
545 void *arg, IB_HANDLE *cq_handle, uint32 *actual_size)
547 return IIBT_IF.Vpi.CreateCQ(hca_handle, requested_size,
548 arg, cq_handle, actual_size);
551 static inline FSTATUS
552 iibt_cq_poll(IB_HANDLE cq_handle, IB_WORK_COMPLETION *wc)
554 return IIBT_IF.Vpi.PollCQ(cq_handle, wc);
557 static inline FSTATUS
558 iibt_cq_rearm(IB_HANDLE cq_handle, IB_CQ_EVENT_SELECT select)
560 return IIBT_IF.Vpi.RearmCQ(cq_handle, select);
563 static inline FSTATUS
564 iibt_cq_destroy(IB_HANDLE cq_handle)
566 return IIBT_IF.Vpi.DestroyCQ(cq_handle);
569 static inline FSTATUS
570 iibt_qp_create(IB_HANDLE hca_handle, IB_QP_ATTRIBUTES_CREATE *create_attr,
571 void *arg, IB_HANDLE *cq_handle,
572 IB_QP_ATTRIBUTES_QUERY *query_attr)
574 return IIBT_IF.Vpi.CreateQP(hca_handle, create_attr, arg, cq_handle,
578 static inline FSTATUS
579 iibt_qp_query(IB_HANDLE qp_handle, IB_QP_ATTRIBUTES_QUERY *query_attr,
582 return IIBT_IF.Vpi.QueryQP(qp_handle, query_attr, arg_ptr);
585 static inline FSTATUS
586 iibt_qp_modify(IB_HANDLE qp_handle, IB_QP_ATTRIBUTES_MODIFY *modify_attr,
587 IB_QP_ATTRIBUTES_QUERY *query_attr)
589 return IIBT_IF.Vpi.ModifyQP(qp_handle, modify_attr, query_attr);
592 static inline FSTATUS
593 iibt_qp_destroy(IB_HANDLE qp_handle)
595 return IIBT_IF.Vpi.DestroyQP(qp_handle);
598 static inline FSTATUS
599 iibt_postrecv(IB_HANDLE qp_handle, IB_WORK_REQ *work_req)
601 return IIBT_IF.Vpi.PostRecv(qp_handle, work_req);
604 static inline FSTATUS
605 iibt_postsend(IB_HANDLE qp_handle, IB_WORK_REQ *work_req)
607 return IIBT_IF.Vpi.PostSend(qp_handle, work_req);
610 static inline FSTATUS
611 iibt_sd_register(IB_HANDLE *sd_handle, CLIENT_CONTROL_PARAMETERS *p)
613 return IIBT_IF.Sdi.Register(sd_handle, p);
616 static inline FSTATUS
617 iibt_sd_deregister(IB_HANDLE sd_handle)
619 return IIBT_IF.Sdi.Deregister(sd_handle);
622 static inline FSTATUS
623 iibt_sd_port_fabric_operation(IB_HANDLE sd_handle, EUI64 port_guid,
624 FABRIC_OPERATION_DATA *fod,
625 PFABRIC_OPERATION_CALLBACK callback,
626 COMMAND_CONTROL_PARAMETERS *p, void *arg)
628 return IIBT_IF.Sdi.PortFabricOperation(sd_handle, port_guid,
629 fod, callback, p, arg);
632 static inline FSTATUS
633 iibt_sd_query_port_fabric_information(IB_HANDLE sd_handle, EUI64 port_guid,
635 PQUERY_CALLBACK callback,
636 COMMAND_CONTROL_PARAMETERS *p, void *arg)
638 return IIBT_IF.Sdi.QueryPortFabricInformation(sd_handle, port_guid,
639 qry, callback, p, arg);
642 static inline IB_HANDLE
643 iibt_cm_create_cep(CM_CEP_TYPE type)
645 return IIBT_IF.Cmi.CmCreateCEP(type);
648 static inline FSTATUS
649 iibt_cm_modify_cep(IB_HANDLE cep, uint32 attr, char* value, uint32 len,
652 return IIBT_IF.Cmi.CmModifyCEP(cep, attr, value, len, offset);
655 static inline FSTATUS
656 iibt_cm_destroy_cep(IB_HANDLE cep_handle)
658 return IIBT_IF.Cmi.CmDestroyCEP(cep_handle);
661 static inline FSTATUS
662 iibt_cm_listen(IB_HANDLE cep, CM_LISTEN_INFO *info,
663 PFN_CM_CALLBACK callback, void *arg)
665 return IIBT_IF.Cmi.CmListen(cep, info, callback, arg);
668 static inline FSTATUS
669 iibt_cm_cancel(IB_HANDLE cep)
671 return IIBT_IF.Cmi.CmCancel(cep);
674 static inline FSTATUS
675 iibt_cm_accept(IB_HANDLE cep,
676 CM_CONN_INFO *send_info, CM_CONN_INFO *recv_info,
677 PFN_CM_CALLBACK callback, void *arg,
680 return IIBT_IF.Cmi.CmAccept(cep,
681 send_info, recv_info,
682 callback, arg, new_cep);
685 static inline FSTATUS
686 iibt_cm_reject(IB_HANDLE cep, CM_REJECT_INFO *rej)
688 return IIBT_IF.Cmi.CmReject(cep, rej);
691 static inline FSTATUS
692 iibt_cm_disconnect(IB_HANDLE cep, CM_DREQUEST_INFO *req,
693 CM_DREPLY_INFO *reply)
695 return IIBT_IF.Cmi.CmDisconnect(cep, req, reply);
698 static inline FSTATUS
699 iibt_cm_connect (IB_HANDLE cep, CM_REQUEST_INFO *req,
700 PFN_CM_CALLBACK callback, void *arg)
702 return IIBT_IF.Cmi.CmConnect (cep, req, callback, arg);
705 static inline int wrq_signals_completion(IB_WORK_REQ *wrq)
707 return wrq->Req.SendRC.Options.s.SignaledCompletion == 1;
711 /******************************************************************************/
713 /* these are purposely avoiding using local vars so they don't increase
714 * stack consumption. */
716 #define kib_peer_addref(peer) do { \
717 LASSERTF(atomic_read(&peer->ibp_refcount) > 0, "%d\n", \
718 atomic_read(&peer->ibp_refcount)); \
719 CDEBUG(D_NET, "++peer[%p] -> "LPX64" (%d)\n", \
720 peer, peer->ibp_nid, atomic_read (&peer->ibp_refcount)); \
721 atomic_inc(&peer->ibp_refcount); \
724 #define kib_peer_decref(peer) do { \
725 LASSERTF(atomic_read(&peer->ibp_refcount) > 0, "%d\n", \
726 atomic_read(&peer->ibp_refcount)); \
727 CDEBUG(D_NET, "--peer[%p] -> "LPX64" (%d)\n", \
728 peer, peer->ibp_nid, atomic_read (&peer->ibp_refcount)); \
729 if (atomic_dec_and_test (&peer->ibp_refcount)) { \
730 CDEBUG (D_NET, "destroying peer "LPX64" %p\n", \
731 peer->ibp_nid, peer); \
732 kibnal_destroy_peer (peer); \
736 /******************************************************************************/
738 static inline struct list_head *
739 kibnal_nid2peerlist (ptl_nid_t nid)
741 unsigned int hash = ((unsigned int)nid) % kibnal_data.kib_peer_hash_size;
743 return (&kibnal_data.kib_peers [hash]);
747 kibnal_peer_active(kib_peer_t *peer)
749 /* Am I in the peer hash table? */
750 return (!list_empty(&peer->ibp_list));
754 kibnal_queue_tx_locked (kib_tx_t *tx, kib_conn_t *conn)
756 /* CAVEAT EMPTOR: tx takes caller's ref on conn */
758 LASSERT (tx->tx_nsp > 0); /* work items set up */
759 LASSERT (tx->tx_conn == NULL); /* only set here */
762 tx->tx_deadline = jiffies + kibnal_tunables.kib_io_timeout * HZ;
763 list_add_tail(&tx->tx_list, &conn->ibc_tx_queue);
766 #define KIBNAL_SERVICE_KEY_MASK (IB_SERVICE_RECORD_COMP_SERVICENAME | \
767 IB_SERVICE_RECORD_COMP_SERVICEDATA8_1 | \
768 IB_SERVICE_RECORD_COMP_SERVICEDATA8_2 | \
769 IB_SERVICE_RECORD_COMP_SERVICEDATA8_3 | \
770 IB_SERVICE_RECORD_COMP_SERVICEDATA8_4 | \
771 IB_SERVICE_RECORD_COMP_SERVICEDATA8_5 | \
772 IB_SERVICE_RECORD_COMP_SERVICEDATA8_6 | \
773 IB_SERVICE_RECORD_COMP_SERVICEDATA8_7 | \
774 IB_SERVICE_RECORD_COMP_SERVICEDATA8_8)
777 kibnal_service_nid_field(IB_SERVICE_RECORD *srv)
779 /* must be consistent with KIBNAL_SERVICE_KEY_MASK */
780 return (__u64 *)srv->ServiceData8;
785 kibnal_set_service_keys(IB_SERVICE_RECORD *srv, ptl_nid_t nid)
787 LASSERT (strlen(IBNAL_SERVICE_NAME) < sizeof(srv->ServiceName));
788 memset (srv->ServiceName, 0, sizeof(srv->ServiceName));
789 strcpy (srv->ServiceName, IBNAL_SERVICE_NAME);
791 *kibnal_service_nid_field(srv) = cpu_to_le64(nid);
796 kibnal_show_rdma_attr (kib_conn_t *conn)
798 struct ib_qp_attribute qp_attr;
801 memset (&qp_attr, 0, sizeof(qp_attr));
802 rc = ib_qp_query(conn->ibc_qp, &qp_attr);
804 CERROR ("Can't get qp attrs: %d\n", rc);
808 CWARN ("RDMA CAPABILITY: write %s read %s\n",
809 (qp_attr.valid_fields & TS_IB_QP_ATTRIBUTE_RDMA_ATOMIC_ENABLE) ?
810 (qp_attr.enable_rdma_write ? "enabled" : "disabled") : "invalid",
811 (qp_attr.valid_fields & TS_IB_QP_ATTRIBUTE_RDMA_ATOMIC_ENABLE) ?
812 (qp_attr.enable_rdma_read ? "enabled" : "disabled") : "invalid");
818 kibnal_page2phys (struct page *p)
820 __u64 page_number = p - mem_map;
822 return (page_number << PAGE_SHIFT);
825 # error "no page->phys"
829 * We rely on tx/rx descriptor alignment to allow us to use the lowest bit
830 * of the work request id as a flag to determine if the completion is for a
831 * transmit or a receive. It seems that that the CQ entry's 'op' field
832 * isn't always set correctly on completions that occur after QP teardown. */
835 kibnal_ptr2wreqid (void *ptr, int isrx)
837 unsigned long lptr = (unsigned long)ptr;
839 LASSERT ((lptr & 1) == 0);
840 return (__u64)(lptr | (isrx ? 1 : 0));
844 kibnal_wreqid2ptr (__u64 wreqid)
846 return (void *)(((unsigned long)wreqid) & ~1UL);
850 kibnal_wreqid_is_rx (__u64 wreqid)
852 return (wreqid & 1) != 0;
856 kibnal_whole_mem(void)
858 return kibnal_data.kib_md.md_handle != NULL;
861 extern kib_peer_t *kibnal_create_peer (ptl_nid_t nid);
862 extern void kibnal_destroy_peer (kib_peer_t *peer);
863 extern int kibnal_del_peer (ptl_nid_t nid, int single_share);
864 extern kib_peer_t *kibnal_find_peer_locked (ptl_nid_t nid);
865 extern void kibnal_unlink_peer_locked (kib_peer_t *peer);
866 extern int kibnal_close_stale_conns_locked (kib_peer_t *peer,
868 extern kib_conn_t *kibnal_create_conn (void);
869 extern void kibnal_put_conn (kib_conn_t *conn);
870 extern void kibnal_destroy_conn (kib_conn_t *conn);
871 void kibnal_listen_callback(IB_HANDLE cep, CM_CONN_INFO *info, void *arg);
873 extern int kibnal_alloc_pages (kib_pages_t **pp, int npages, int access);
874 extern void kibnal_free_pages (kib_pages_t *p);
876 extern void kibnal_check_sends (kib_conn_t *conn);
877 extern void kibnal_close_conn_locked (kib_conn_t *conn, int error);
878 extern void kibnal_destroy_conn (kib_conn_t *conn);
879 extern int kibnal_thread_start (int (*fn)(void *arg), void *arg);
880 extern int kibnal_scheduler(void *arg);
881 extern int kibnal_connd (void *arg);
882 extern void kibnal_init_tx_msg (kib_tx_t *tx, int type, int body_nob);
883 extern void kibnal_close_conn (kib_conn_t *conn, int why);
884 extern void kibnal_start_active_rdma (int type, int status,
885 kib_rx_t *rx, lib_msg_t *libmsg,
887 struct iovec *iov, ptl_kiov_t *kiov,
888 size_t offset, size_t nob);
890 void kibnal_ca_async_callback (void *ca_arg, IB_EVENT_RECORD *ev);
891 void kibnal_ca_callback (void *ca_arg, void *cq_arg);