2 * Copyright (C) 2009-2012 Cray, Inc.
4 * Author: Nic Henke <nic@cray.com>
6 * This file is part of Lustre, http://www.lustre.org.
8 * Lustre is free software; you can redistribute it and/or
9 * modify it under the terms of version 2 of the GNU General Public
10 * License as published by the Free Software Foundation.
12 * Lustre is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with Lustre; if not, write to the Free Software
19 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 #ifndef _GNILND_API_WRAP_H
23 #define _GNILND_API_WRAP_H
25 /* LNet is allocated failure locations 0xe000 to 0xffff */
27 /* GNILND has 0xf0XX */
28 #define CFS_FAIL_GNI 0xf000
29 #define CFS_FAIL_GNI_PHYS_MAP 0xf001
30 #define CFS_FAIL_GNI_VIRT_MAP 0xf002
31 #define CFS_FAIL_GNI_GET_UNMAP 0xf003
32 #define CFS_FAIL_GNI_PUT_UNMAP 0xf004
33 #define CFS_FAIL_GNI_MAP_TX 0xf005
34 #define CFS_FAIL_GNI_SMSG_SEND 0xf006
35 #define CFS_FAIL_GNI_CLOSE_SEND 0xf007
36 #define CFS_FAIL_GNI_CDM_CREATE 0xf008
37 #define CFS_FAIL_GNI_CDM_DESTROY 0xf009
38 #define CFS_FAIL_GNI_CDM_ATTACH 0xf00a
39 #define CFS_FAIL_GNI_CQ_CREATE 0xf00b
40 #define CFS_FAIL_GNI_CQ_DESTROY 0xf00c
41 #define CFS_FAIL_GNI_EP_BIND 0xf00d
42 #define CFS_FAIL_GNI_EP_UNBIND 0xf00e
43 #define CFS_FAIL_GNI_EP_SET_EVDATA 0xf00f
44 #define CFS_FAIL_GNI_SMSG_INIT 0xf010
45 #define CFS_FAIL_GNI_SMSG_RELEASE 0xf011
46 #define CFS_FAIL_GNI_POST_RDMA 0xf012
47 #define CFS_FAIL_GNI_GET_COMPLETED 0xf013
48 #define CFS_FAIL_GNI_EP_DESTROY 0xf015
49 #define CFS_FAIL_GNI_VIRT_UNMAP 0xf016
50 #define CFS_FAIL_GNI_MDD_RELEASE 0xf017
51 #define CFS_FAIL_GNI_NOOP_SEND 0xf018
52 #define CFS_FAIL_GNI_ERR_SUBSCRIBE 0xf01a
53 #define CFS_FAIL_GNI_QUIESCE_RACE 0xf01b
54 #define CFS_FAIL_GNI_DG_TERMINATE 0xf01c
55 #define CFS_FAIL_GNI_REG_QUIESCE 0xf01d
56 #define CFS_FAIL_GNI_IN_QUIESCE 0xf01e
57 #define CFS_FAIL_GNI_DELAY_RDMA 0xf01f
58 #define CFS_FAIL_GNI_SR_DOWN_RACE 0xf020
59 #define CFS_FAIL_GNI_ALLOC_TX 0xf021
60 #define CFS_FAIL_GNI_FMABLK_AVAIL 0xf022
61 #define CFS_FAIL_GNI_EP_CREATE 0xf023
62 #define CFS_FAIL_GNI_CQ_GET_EVENT 0xf024
63 #define CFS_FAIL_GNI_PROBE 0xf025
64 #define CFS_FAIL_GNI_EP_TEST 0xf026
65 #define CFS_FAIL_GNI_CONNREQ_DROP 0xf027
66 #define CFS_FAIL_GNI_CONNREQ_PROTO 0xf028
67 #define CFS_FAIL_GNI_CONND_PILEUP 0xf029
68 #define CFS_FAIL_GNI_PHYS_SETUP 0xf02a
69 #define CFS_FAIL_GNI_FIND_TARGET 0xf02b
70 #define CFS_FAIL_GNI_WC_DGRAM_FREE 0xf02c
71 #define CFS_FAIL_GNI_DROP_CLOSING 0xf02d
72 #define CFS_FAIL_GNI_RX_CLOSE_CLOSING 0xf02e
73 #define CFS_FAIL_GNI_RX_CLOSE_CLOSED 0xf02f
74 #define CFS_FAIL_GNI_EP_POST 0xf030
75 #define CFS_FAIL_GNI_PACK_SRCNID 0xf031
76 #define CFS_FAIL_GNI_PACK_DSTNID 0xf032
77 #define CFS_FAIL_GNI_PROBE_WAIT 0xf033
78 #define CFS_FAIL_GNI_SMSG_CKSUM1 0xf034
79 #define CFS_FAIL_GNI_SMSG_CKSUM2 0xf035
80 #define CFS_FAIL_GNI_SMSG_CKSUM3 0xf036
81 #define CFS_FAIL_GNI_DROP_DESTROY_EP 0xf037
82 #define CFS_FAIL_GNI_SMSG_GETNEXT 0xf038
83 #define CFS_FAIL_GNI_FINISH_PURG 0xf039
84 #define CFS_FAIL_GNI_PURG_REL_DELAY 0xf03a
85 #define CFS_FAIL_GNI_DONT_NOTIFY 0xf03b
86 #define CFS_FAIL_GNI_VIRT_SMALL_MAP 0xf03c
87 #define CFS_FAIL_GNI_DELAY_RDMAQ 0xf03d
88 #define CFS_FAIL_GNI_PAUSE_SHUTDOWN 0xf03e
89 #define CFS_FAIL_GNI_PAUSE_DGRAM_COMP 0xf03f
90 #define CFS_FAIL_GNI_NET_LOOKUP 0xf040
91 #define CFS_FAIL_GNI_RECV_TIMEOUT 0xf041
92 #define CFS_FAIL_GNI_SEND_TIMEOUT 0xf042
93 #define CFS_FAIL_GNI_ONLY_NOOP 0xf043
94 #define CFS_FAIL_GNI_FINISH_PURG2 0xf044
95 #define CFS_FAIL_GNI_RACE_RESET 0xf045
96 #define CFS_FAIL_GNI_GNP_CONNECTING1 0xf046
97 #define CFS_FAIL_GNI_GNP_CONNECTING2 0xf047
98 #define CFS_FAIL_GNI_GNP_CONNECTING3 0xf048
99 #define CFS_FAIL_GNI_SCHEDULE_COMPLETE 0xf049
100 #define CFS_FAIL_GNI_PUT_ACK_AGAIN 0xf050
101 #define CFS_FAIL_GNI_GET_REQ_AGAIN 0xf051
102 #define CFS_FAIL_GNI_SCHED_DEADLINE 0xf052
103 #define CFS_FAIL_GNI_DGRAM_DEADLINE 0xf053
104 #define CFS_FAIL_GNI_DGRAM_DROP_TX 0xf054
105 #define CFS_FAIL_GNI_RDMA_CQ_ERROR 0xf055
109 _kgnilnd_api_rc_lbug(const char *rcstr, int rc, struct libcfs_debug_msg_data *data,
110 const char *fmt, ...)
111 __attribute__ ((format (printf, 4, 5)));
113 #define kgnilnd_api_rc_lbug(msgdata, rc, fmt, a...) \
115 CFS_CHECK_STACK(msgdata, D_ERROR, NULL); \
116 /* we don't mask this - it is always at D_ERROR */ \
117 _kgnilnd_api_rc_lbug(kgnilnd_api_rc2str(rc), (rc), msgdata, fmt, ##a); \
120 #define DO_RETCODE(x) case x: return #x;
121 static inline const char *
122 kgnilnd_api_rc2str(gni_return_t rrc)
126 DO_RETCODE(GNI_RC_SUCCESS)
127 DO_RETCODE(GNI_RC_NOT_DONE);
128 DO_RETCODE(GNI_RC_INVALID_PARAM);
129 DO_RETCODE(GNI_RC_ERROR_RESOURCE);
130 DO_RETCODE(GNI_RC_TIMEOUT);
131 DO_RETCODE(GNI_RC_PERMISSION_ERROR);
132 DO_RETCODE(GNI_RC_DESCRIPTOR_ERROR);
133 DO_RETCODE(GNI_RC_ALIGNMENT_ERROR);
134 DO_RETCODE(GNI_RC_INVALID_STATE);
135 DO_RETCODE(GNI_RC_NO_MATCH);
136 DO_RETCODE(GNI_RC_SIZE_ERROR);
137 DO_RETCODE(GNI_RC_TRANSACTION_ERROR);
138 DO_RETCODE(GNI_RC_ILLEGAL_OP);
139 DO_RETCODE(GNI_RC_ERROR_NOMEM);
145 /* log an error and LBUG for unhandled rc from gni api function
146 * the fmt should be something like:
147 * gni_api_call(arg1, arg2, arg3)
150 /* apick_fn and apick_fmt should be defined for each site */
154 #define GNILND_API_RC_LBUG(args...) \
156 LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL); \
157 kgnilnd_api_rc_lbug(&msgdata, rrc, apick_fn"("apick_fmt")", ##args); \
160 #define GNILND_API_SWBUG(args...) \
162 CERROR("likely SOFTWARE BUG "apick_fn"("apick_fmt") rc %s\n", \
163 ##args, kgnilnd_api_rc2str(rrc)); \
166 #define GNILND_API_EINVAL(args...) \
168 CERROR("invalid parameter to "apick_fn"("apick_fmt") rc %s\n", \
169 ##args, kgnilnd_api_rc2str(rrc)); \
172 #define GNILND_API_RESOURCE(args...) \
174 CERROR("no resources for "apick_fn"("apick_fmt") rc %s\n", \
175 ##args, kgnilnd_api_rc2str(rrc)); \
178 #define GNILND_API_BUSY(args...) \
180 CERROR("resources busy for "apick_fn"("apick_fmt") rc %s\n", \
181 ##args, kgnilnd_api_rc2str(rrc)); \
184 #undef DEBUG_SMSG_CREDITS
185 #ifdef DEBUG_SMSG_CREDITS
186 #define CRAY_CONFIG_GHAL_GEMINI
187 #include <gni_priv.h>
188 #define GNIDBG_SMSG_CREDS(level, conn) \
190 gni_ep_smsg_mbox_t *smsg = conn->gnc_ephandle->smsg; \
191 CDEBUG(level, "SMSGDBG: conn %p mcred %d/%d bcred %d/%d " \
192 "s_seq %d/%d/%d r_seq %d/%d/%d retr %d\n", \
193 conn, smsg->mbox_credits, smsg->back_mbox_credits, \
194 smsg->buffer_credits, smsg->back_buffer_credits, \
195 smsg->s_seqno, smsg->s_seqno_back_mbox_credits, \
196 smsg->s_seqno_back_buffer_credits, smsg->r_seqno, \
197 smsg->r_seqno_back_mbox_credits, \
198 smsg->r_seqno_back_buffer_credits, smsg->retransmit_count); \
201 #define GNIDBG_SMSG_CREDS(level, conn) do {} while(0)
204 /* these are all wrappers around gni_XXX functions.
205 * This allows us to handle all the return codes and api checks without
206 * dirtying up the logic code */
208 /* TODO: RETURN wrapper that translates integer to GNI API RC string */
210 #define apick_fn "kgnilnd_cdm_create"
211 #define apick_fmt "%u, %u, %u, %u, 0x%p"
212 static inline gni_return_t kgnilnd_cdm_create(
217 OUT gni_cdm_handle_t *cdm_hndl
222 /* Error injection */
223 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CDM_CREATE)) {
224 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
226 rrc = gni_cdm_create(inst_id, ptag, cookie, modes, cdm_hndl);
232 case GNI_RC_ERROR_RESOURCE:
233 case GNI_RC_INVALID_PARAM:
234 /* Try to bail gracefully */
236 inst_id, ptag, cookie, modes, cdm_hndl);
240 inst_id, ptag, cookie, modes, cdm_hndl);
242 /* LBUG never returns, but just for style and consistency */
251 #define apick_fn "kgnilnd_cdm_attach"
252 #define apick_fmt "0x%p, %u, 0x%p, 0x%p"
253 static inline gni_return_t kgnilnd_cdm_attach(
254 IN gni_cdm_handle_t cdm_hndl,
255 IN uint32_t device_id,
256 OUT uint32_t *local_addr,
257 OUT gni_nic_handle_t *nic_hndl
262 /* Error injection */
263 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CDM_ATTACH)) {
264 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
266 rrc = gni_cdm_attach(cdm_hndl, device_id, local_addr, nic_hndl);
272 case GNI_RC_NO_MATCH:
273 case GNI_RC_INVALID_PARAM:
275 cdm_hndl, device_id, local_addr, nic_hndl);
277 case GNI_RC_ERROR_RESOURCE:
278 case GNI_RC_INVALID_STATE:
280 cdm_hndl, device_id, local_addr, nic_hndl);
284 cdm_hndl, device_id, local_addr, nic_hndl);
286 /* LBUG never returns, but just for style and consistency */
294 #define apick_fn "kgnilnd_cdm_destroy"
295 #define apick_fmt "0x%p"
296 static inline gni_return_t kgnilnd_cdm_destroy(
297 IN gni_cdm_handle_t cdm_hndl
302 /* Error injection */
303 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_DESTROY)) {
304 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
306 rrc = gni_cdm_destroy(
313 case GNI_RC_INVALID_PARAM:
321 /* LBUG never returns, but just for style and consistency */
329 #define apick_fn "kgnilnd_subscribe_errors"
330 #define apick_fmt "0x%p,%x,%u,0x%p,0x%p,0x%p"
331 static inline gni_return_t kgnilnd_subscribe_errors(
332 IN gni_nic_handle_t nic_handle,
333 IN gni_error_mask_t mask,
334 IN uint32_t EEQ_size,
335 IN void (*EQ_new_event)(gni_err_handle_t),
336 IN void (*app_crit_err)(gni_err_handle_t),
337 OUT gni_err_handle_t *err_handle
342 /* Error injection */
343 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_ERR_SUBSCRIBE)) {
344 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
346 rrc = gni_subscribe_errors(
347 nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
354 case GNI_RC_INVALID_PARAM:
356 nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
359 case GNI_RC_ERROR_RESOURCE:
361 nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
366 nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
368 /* LBUG never returns, but just for style and consistency */
376 #define apick_fn "kgnilnd_release_errors"
377 #define apick_fmt "0x%p"
378 static inline gni_return_t kgnilnd_release_errors(
379 IN gni_err_handle_t err_handle
384 rrc = gni_release_errors(
390 case GNI_RC_INVALID_PARAM:
391 case GNI_RC_NOT_DONE:
398 /* LBUG never returns, but just for style and consistency */
406 #define apick_fn "kgnilnd_set_quiesce_callback"
407 #define apick_fmt "0x%p,0x%p"
408 static inline gni_return_t kgnilnd_set_quiesce_callback(
409 IN gni_nic_handle_t nic_handle,
410 IN void (*qsce_func)(gni_nic_handle_t, uint64_t msecs)
415 /* Error injection */
416 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_REG_QUIESCE)) {
417 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
419 rrc = gni_set_quiesce_callback(
420 nic_handle, qsce_func);
426 case GNI_RC_INVALID_STATE:
427 case GNI_RC_INVALID_PARAM:
429 nic_handle, qsce_func);
433 nic_handle, qsce_func);
434 /* LBUG never returns, but just for style and consistency */
442 #define apick_fn "kgnilnd_get_quiesce_status"
443 #define apick_fmt "0x%p"
444 static inline gni_return_t kgnilnd_get_quiesce_status(
445 IN gni_nic_handle_t nic_handle
450 /* this has weird RC -
451 * 0 - quiesce not in progress
452 * 1 - quiesce is turned on
455 /* Error injection */
456 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_IN_QUIESCE)) {
459 rrc = gni_get_quiesce_status(
470 /* LBUG never returns, but just for style and consistency */
478 #define apick_fn "kgnilnd_cq_create"
479 #define apick_fmt "0x%p, %u, %u, 0x%p, %#llx, 0x%p"
480 static inline gni_return_t kgnilnd_cq_create(
481 IN gni_nic_handle_t nic_hndl,
482 IN uint32_t entry_count,
483 IN uint32_t delay_index,
484 IN gni_cq_event_hndlr_f *event_handler,
485 IN uint64_t usr_event_data,
486 OUT gni_cq_handle_t *cq_hndl
491 /* Error injection */
492 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_CREATE)) {
493 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
496 nic_hndl, entry_count, delay_index, event_handler,
497 usr_event_data, cq_hndl);
503 case GNI_RC_INVALID_PARAM:
505 nic_hndl, entry_count, delay_index, event_handler,
506 usr_event_data, cq_hndl);
508 case GNI_RC_ERROR_RESOURCE:
510 nic_hndl, entry_count, delay_index, event_handler,
511 usr_event_data, cq_hndl);
515 nic_hndl, entry_count, delay_index, event_handler,
516 usr_event_data, cq_hndl);
518 /* LBUG never returns, but just for style and consistency */
526 #define apick_fn "kgnilnd_cq_destroy"
527 #define apick_fmt "0x%p"
528 static inline gni_return_t kgnilnd_cq_destroy(
529 IN gni_cq_handle_t cq_hndl
534 /* Error injection */
535 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_DESTROY)) {
536 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
539 rrc = gni_cq_destroy(
546 case GNI_RC_INVALID_PARAM:
550 case GNI_RC_ERROR_RESOURCE:
558 /* LBUG never returns, but just for style and consistency */
566 #define apick_fn "kgnilnd_cq_get_event"
567 #define apick_fmt "0x%p, 0x%p"
568 static inline gni_return_t kgnilnd_cq_get_event(
569 IN gni_cq_handle_t cq_hndl,
570 OUT gni_cq_entry_t *event_data
575 /* no error injection - CQs are touchy about the data.
576 * where appropriate, we'll do this on the CQs that should be able to
577 * handle the various errors */
578 rrc = gni_cq_get_event(
579 cq_hndl, event_data);
583 case GNI_RC_NOT_DONE:
584 case GNI_RC_TRANSACTION_ERROR:
586 case GNI_RC_ERROR_RESOURCE:
587 LASSERTF(GNI_CQ_OVERRUN(*event_data),
588 "kgni returned ERROR_RESOURCE but cq_hndl 0x%p is not "
589 "overrun\n", cq_hndl);
591 case GNI_RC_INVALID_PARAM:
593 cq_hndl, event_data);
597 cq_hndl, event_data);
599 /* LBUG never returns, but just for style and consistency */
607 #define apick_fn "kgnilnd_smsg_init"
608 #define apick_fmt "0x%p, 0x%p, 0x%p"
609 static inline gni_return_t kgnilnd_smsg_init(
610 IN gni_ep_handle_t ep_hndl,
611 IN gni_smsg_attr_t *local_smsg_attr,
612 IN gni_smsg_attr_t *remote_smsg_attr
617 /* Error injection */
618 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_INIT)) {
619 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_RESOURCE;
622 ep_hndl, local_smsg_attr, remote_smsg_attr);
626 /* both of these are OK, upper SW needs to handle */
628 case GNI_RC_NOT_DONE:
630 case GNI_RC_INVALID_PARAM:
631 case GNI_RC_INVALID_STATE:
633 ep_hndl, local_smsg_attr, remote_smsg_attr);
635 case GNI_RC_ERROR_RESOURCE:
637 ep_hndl, local_smsg_attr, remote_smsg_attr);
641 ep_hndl, local_smsg_attr, remote_smsg_attr);
643 /* LBUG never returns, but just for style and consistency */
651 #define apick_fn "kgnilnd_smsg_send"
652 #define apick_fmt "0x%p, 0x%p, %d, 0x%p, %u %u"
653 static inline gni_return_t kgnilnd_smsg_send(
654 IN gni_ep_handle_t ep_hndl,
656 IN uint32_t header_length,
658 IN uint32_t data_length,
664 /* Error injection */
665 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_SEND)) {
666 if (cfs_fail_loc & CFS_FAIL_RAND) {
667 rrc = GNI_RC_NOT_DONE;
669 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
673 ep_hndl, header, header_length, data, data_length, msg_id);
677 /* both of these are OK, upper SW needs to handle */
679 case GNI_RC_NOT_DONE:
681 case GNI_RC_INVALID_PARAM:
683 ep_hndl, header, header_length, data, data_length, msg_id);
685 case GNI_RC_ERROR_RESOURCE:
687 ep_hndl, header, header_length, data, data_length, msg_id);
691 ep_hndl, header, header_length, data, data_length, msg_id);
693 /* LBUG never returns, but just for style and consistency */
701 #define apick_fn "kgnilnd_smsg_getnext"
702 #define apick_fmt "0x%p,0x%p"
703 static inline gni_return_t kgnilnd_smsg_getnext(
704 IN gni_ep_handle_t ep_hndl,
710 /* Error injection */
711 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_RELEASE)) {
712 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
714 rrc = gni_smsg_getnext(
719 /* both of these are OK, upper SW needs to handle */
721 case GNI_RC_NOT_DONE:
722 case GNI_RC_INVALID_STATE:
724 case GNI_RC_INVALID_PARAM:
732 /* LBUG never returns, but just for style and consistency */
740 #define apick_fn "kgnilnd_smsg_release"
741 #define apick_fmt "0x%p"
742 static inline gni_return_t kgnilnd_smsg_release(
743 IN gni_ep_handle_t ep_hndl
748 /* Error injection */
749 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_RELEASE)) {
750 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
752 rrc = gni_smsg_release(
757 /* both of these are OK, upper SW needs to handle */
759 case GNI_RC_NOT_DONE:
761 case GNI_RC_INVALID_PARAM:
769 /* LBUG never returns, but just for style and consistency */
777 #define apick_fn "kgnilnd_ep_create"
778 #define apick_fmt "0x%p, 0x%p, 0x%p"
779 static inline gni_return_t kgnilnd_ep_create(
780 IN gni_nic_handle_t nic_hndl,
781 IN gni_cq_handle_t src_cq_hndl,
782 OUT gni_ep_handle_t *ep_hndl
787 /* error injection */
788 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_CREATE)) {
789 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_NOMEM;
792 nic_hndl, src_cq_hndl, ep_hndl);
798 case GNI_RC_INVALID_PARAM:
800 nic_hndl, src_cq_hndl, ep_hndl);
802 case GNI_RC_ERROR_NOMEM:
804 nic_hndl, src_cq_hndl, ep_hndl);
808 nic_hndl, src_cq_hndl, ep_hndl);
810 /* lbug never returns, but just for style and consistency */
818 #define apick_fn "kgnilnd_ep_bind"
819 #define apick_fmt "0x%p, %x, %x"
820 static inline gni_return_t kgnilnd_ep_bind(
821 IN gni_ep_handle_t ep_hndl,
822 IN uint32_t remote_addr,
823 IN uint32_t remote_id
828 /* error injection */
829 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_BIND)) {
830 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
833 ep_hndl, remote_addr, remote_id);
837 /* both of these are ok, upper sw needs to handle */
839 case GNI_RC_NOT_DONE:
841 case GNI_RC_INVALID_PARAM:
843 ep_hndl, remote_addr, remote_id);
847 ep_hndl, remote_addr, remote_id);
849 /* lbug never returns, but just for style and consistency */
857 #define apick_fn "kgnilnd_ep_set_eventdata"
858 #define apick_fmt "0x%p, %x, %x"
859 static inline gni_return_t kgnilnd_ep_set_eventdata(
860 IN gni_ep_handle_t ep_hndl,
861 IN uint32_t local_event,
862 IN uint32_t remote_event
867 /* Error injection */
868 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_SET_EVDATA)) {
869 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
871 rrc = gni_ep_set_eventdata(
872 ep_hndl, local_event, remote_event);
878 case GNI_RC_INVALID_PARAM:
880 ep_hndl, local_event, remote_event);
884 ep_hndl, local_event, remote_event);
886 /* LBUG never returns, but just for style and consistency */
894 #define apick_fn "kgnilnd_ep_unbind"
895 #define apick_fmt "0x%p"
896 static inline gni_return_t kgnilnd_ep_unbind(
897 IN gni_ep_handle_t ep_hndl
902 /* Error injection */
903 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_UNBIND)) {
904 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
911 /* both of these are OK, upper SW needs to handle */
912 case GNI_RC_NOT_DONE:
915 case GNI_RC_INVALID_PARAM:
923 /* LBUG never returns, but just for style and consistency */
931 #define apick_fn "kgnilnd_ep_destroy"
932 #define apick_fmt "0x%p"
933 static inline gni_return_t kgnilnd_ep_destroy(
934 IN gni_ep_handle_t ep_hndl
939 /* Error injection */
940 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_DESTROY)) {
941 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
943 rrc = gni_ep_destroy(
950 case GNI_RC_INVALID_PARAM:
958 /* LBUG never returns, but just for style and consistency */
966 #define apick_fn "kgnilnd_ep_postdata_w_id"
967 #define apick_fmt "0x%p, 0x%p, %d, 0x%p, %d, %llu"
968 static inline gni_return_t kgnilnd_ep_postdata_w_id(
969 IN gni_ep_handle_t ep_hndl,
971 IN uint16_t data_len,
973 IN uint16_t buf_size,
974 IN uint64_t datagram_id
979 /* Error injection */
980 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_POST)) {
981 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_SIZE_ERROR;
983 rrc = gni_ep_postdata_w_id(
984 ep_hndl, in_data, data_len, out_buf, buf_size,
990 case GNI_RC_ERROR_NOMEM:
991 case GNI_RC_ERROR_RESOURCE:
993 case GNI_RC_INVALID_PARAM:
994 case GNI_RC_SIZE_ERROR:
996 ep_hndl, in_data, data_len, out_buf, buf_size,
1001 ep_hndl, in_data, data_len, out_buf, buf_size,
1004 /* LBUG never returns, but just for style and consistency */
1012 #define apick_fn "kgnilnd_ep_postdata_test_by_id"
1013 #define apick_fmt "0x%p, %llu, 0x%p, 0x%p, 0x%p"
1014 static inline gni_return_t kgnilnd_ep_postdata_test_by_id(
1015 IN gni_ep_handle_t ep_hndl,
1016 IN uint64_t datagram_id,
1017 OUT gni_post_state_t *post_state,
1018 OUT uint32_t *remote_addr,
1019 OUT uint32_t *remote_id
1024 /* Error injection */
1025 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_TEST)) {
1026 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_NOMEM;
1028 rrc = gni_ep_postdata_test_by_id(
1029 ep_hndl, datagram_id, post_state, remote_addr,
1032 /* we want to lie, but we need to do the actual work first
1033 * so we don't keep getting the event saying a dgram is ready */
1034 if (rrc == GNI_RC_SUCCESS && CFS_FAIL_CHECK(CFS_FAIL_GNI_DG_TERMINATE)) {
1035 /* don't use fail_val, allows us to do FAIL_SOME */
1036 *post_state = GNI_POST_TERMINATED;
1041 case GNI_RC_SUCCESS:
1042 case GNI_RC_NO_MATCH:
1044 case GNI_RC_SIZE_ERROR:
1045 case GNI_RC_INVALID_PARAM:
1047 ep_hndl, datagram_id, post_state, remote_addr,
1050 case GNI_RC_ERROR_NOMEM:
1051 GNILND_API_RESOURCE(
1052 ep_hndl, datagram_id, post_state, remote_addr,
1057 ep_hndl, datagram_id, post_state, remote_addr,
1060 /* LBUG never returns, but just for style and consistency */
1068 #define apick_fn "kgnilnd_ep_postdata_cancel_by_id"
1069 #define apick_fmt "0x%p, %llu"
1070 static inline gni_return_t kgnilnd_ep_postdata_cancel_by_id(
1071 IN gni_ep_handle_t ep_hndl,
1072 IN uint64_t datagram_id
1077 /* no error injection as the only thing we'd do is LBUG */
1079 rrc = gni_ep_postdata_cancel_by_id(
1080 ep_hndl, datagram_id);
1083 case GNI_RC_SUCCESS:
1084 case GNI_RC_NO_MATCH:
1086 case GNI_RC_INVALID_PARAM:
1088 ep_hndl, datagram_id);
1092 ep_hndl, datagram_id);
1094 /* LBUG never returns, but just for style and consistency */
1102 #define apick_fn "kgnilnd_postdata_probe_by_id"
1103 #define apick_fmt "0x%p, 0x%p"
1104 static inline gni_return_t kgnilnd_postdata_probe_by_id(
1105 IN gni_nic_handle_t nic_hndl,
1106 OUT uint64_t *datagram_id
1111 /* Error injection */
1112 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PROBE)) {
1113 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NO_MATCH;
1115 rrc = gni_postdata_probe_by_id(
1116 nic_hndl, datagram_id);
1120 case GNI_RC_SUCCESS:
1121 case GNI_RC_NO_MATCH:
1123 case GNI_RC_INVALID_PARAM:
1125 nic_hndl, datagram_id);
1129 nic_hndl, datagram_id);
1131 /* LBUG never returns, but just for style and consistency */
1139 #define apick_fn "kgnilnd_postdata_probe_wait_by_id"
1140 #define apick_fmt "0x%p, %d, 0x%p"
1141 static inline gni_return_t kgnilnd_postdata_probe_wait_by_id(
1142 IN gni_nic_handle_t nic_hndl,
1143 IN uint32_t timeout,
1144 OUT uint64_t *datagram_id
1149 /* Error injection */
1150 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PROBE_WAIT)) {
1151 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_TIMEOUT;
1153 rrc = gni_postdata_probe_wait_by_id(
1154 nic_hndl, timeout, datagram_id);
1158 case GNI_RC_SUCCESS:
1159 case GNI_RC_TIMEOUT:
1161 case GNI_RC_INVALID_PARAM:
1163 nic_hndl, timeout, datagram_id);
1167 nic_hndl, timeout, datagram_id);
1169 /* LBUG never returns, but just for style and consistency */
1177 #define apick_fn "kgnilnd_post_rdma"
1178 #define apick_fmt "0x%p, 0x%p"
1179 static inline gni_return_t kgnilnd_post_rdma(
1180 IN gni_ep_handle_t ep_hndl,
1181 IN gni_post_descriptor_t *post_descr
1186 /* Error injection */
1187 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_POST_RDMA)) {
1188 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
1190 rrc = gni_post_rdma(
1191 ep_hndl, post_descr);
1195 case GNI_RC_SUCCESS:
1197 case GNI_RC_ALIGNMENT_ERROR:
1198 case GNI_RC_INVALID_PARAM:
1200 ep_hndl, post_descr);
1202 case GNI_RC_ERROR_RESOURCE:
1203 CDEBUG(D_NET, "no resources for kgnilnd_post_rdma (0x%p, 0x%p)"
1204 " rc %s\n", ep_hndl, post_descr,
1205 kgnilnd_api_rc2str(rrc));
1209 ep_hndl, post_descr);
1211 /* LBUG never returns, but just for style and consistency */
1219 #define apick_fn "kgnilnd_get_completed"
1220 #define apick_fmt "0x%p,%#llx,0x%p"
1221 static inline gni_return_t kgnilnd_get_completed(
1222 IN gni_cq_handle_t cq_hndl,
1223 IN gni_cq_entry_t event_data,
1224 OUT gni_post_descriptor_t **post_descr
1230 rrc = gni_get_completed(cq_hndl, event_data, post_descr);
1233 case GNI_RC_TRANSACTION_ERROR:
1234 case GNI_RC_SUCCESS:
1236 case GNI_RC_DESCRIPTOR_ERROR:
1237 case GNI_RC_INVALID_PARAM:
1238 GNILND_API_SWBUG(cq_hndl, event_data, post_descr);
1241 GNILND_API_RC_LBUG(cq_hndl, event_data, post_descr);
1242 /* LBUG never returns, but just for style and consistency */
1246 /* Error injection - we need a valid desc, so let kgni give us one
1248 if (rrc == GNI_RC_SUCCESS &&
1249 (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED))) {
1250 /* We only trigger TRANSACTION_ERROR for now */
1251 gni_post_descriptor_t *desc;
1252 rrc = GNI_RC_TRANSACTION_ERROR;
1255 /* recoverable decision made from cfs_fail_val in
1256 * kgnilnd_cq_error_str and
1257 * kgnilnd_cq_error_recoverable */
1264 #define apick_fn "kgnilnd_cq_error_str"
1265 #define apick_fmt "%#llx,0x%p,%d"
1266 static inline gni_return_t kgnilnd_cq_error_str(
1267 IN gni_cq_entry_t entry,
1274 /* Error injection - set string if we injected a
1275 * TRANSACTION_ERROR earlier */
1276 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED)) {
1277 /* if we just set persistent error, we can't ever
1278 * break in via ssh to clear, so use a count > 10 to indicate fatal */
1279 sprintf(buffer, "INJECT:%s", cfs_fail_val > 10 ?
1280 "FATAL" : "RECOVERABLE");
1281 rrc = GNI_RC_SUCCESS;
1283 rrc = gni_cq_error_str(
1284 entry, buffer, len);
1288 case GNI_RC_SUCCESS:
1290 case GNI_RC_SIZE_ERROR:
1291 case GNI_RC_INVALID_PARAM:
1293 entry, buffer, len);
1294 /* give them something to use */
1295 snprintf(buffer, len, "UNDEF:UNDEF");
1299 entry, buffer, len);
1301 /* LBUG never returns, but just for style and consistency */
1309 #define apick_fn "kgnilnd_cq_error_recoverable"
1310 #define apick_fmt "%#llx,0x%p"
1311 static inline gni_return_t kgnilnd_cq_error_recoverable(
1312 IN gni_cq_entry_t entry,
1313 IN uint32_t *recoverable
1318 /* Error injection - set string if we injected a
1319 * TRANSACTION_ERROR earlier */
1320 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED)) {
1321 *recoverable = cfs_fail_val > 10 ? 0 : 1;
1322 rrc = GNI_RC_SUCCESS;
1324 rrc = gni_cq_error_recoverable(
1325 entry, recoverable);
1329 case GNI_RC_SUCCESS:
1331 case GNI_RC_INVALID_STATE:
1332 case GNI_RC_INVALID_PARAM:
1334 entry, recoverable);
1339 entry, recoverable);
1341 /* LBUG never returns, but just for style and consistency */
1349 #define apick_fn "kgnilnd_mem_register_segments"
1350 #define apick_fmt "0x%p,0x%p,%u,0x%p,%x,0x%p"
1351 static inline gni_return_t
1352 kgnilnd_mem_register_segments(
1353 IN gni_nic_handle_t nic_hndl,
1354 IN gni_mem_segment_t *mem_segments,
1355 IN uint32_t segments_cnt,
1356 IN gni_cq_handle_t dst_cq_hndl,
1358 OUT gni_mem_handle_t *mem_hndl
1363 /* Error injection */
1364 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PHYS_MAP)) {
1365 rrc = GNI_RC_ERROR_RESOURCE;
1367 rrc = gni_mem_register_segments(
1368 nic_hndl, mem_segments, segments_cnt,
1369 dst_cq_hndl, flags, mem_hndl);
1373 case GNI_RC_SUCCESS:
1374 case GNI_RC_ERROR_RESOURCE:
1376 case GNI_RC_INVALID_PARAM:
1378 nic_hndl, mem_segments, segments_cnt,
1379 dst_cq_hndl, flags, mem_hndl);
1383 nic_hndl, mem_segments, segments_cnt,
1384 dst_cq_hndl, flags, mem_hndl);
1386 /* LBUG never returns, but just for style and consistency */
1394 #define apick_fn "kgnilnd_mem_register"
1395 #define apick_fmt "0x%p,%#llx,%#llx0x%p,%u,0x%p"
1396 static inline gni_return_t kgnilnd_mem_register(
1397 IN gni_nic_handle_t nic_hndl,
1398 IN uint64_t address,
1400 IN gni_cq_handle_t dst_cq_hndl,
1402 OUT gni_mem_handle_t *mem_hndl
1407 /* Error injection */
1408 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_MAP)) {
1409 rrc = GNI_RC_ERROR_RESOURCE;
1410 } else if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_SMALL_MAP) &&
1411 length <= *kgnilnd_tunables.kgn_max_immediate) {
1412 rrc = GNI_RC_INVALID_PARAM;
1414 rrc = gni_mem_register(
1415 nic_hndl, address, length,
1416 dst_cq_hndl, flags, mem_hndl);
1419 /* gni_mem_register may return GNI_RC_ERROR_NOMEM under memory
1420 * pressure but the upper layers only know about resource errors
1422 if (rrc == GNI_RC_ERROR_NOMEM) {
1423 rrc = GNI_RC_ERROR_RESOURCE;
1427 case GNI_RC_SUCCESS:
1428 case GNI_RC_ERROR_RESOURCE:
1430 case GNI_RC_INVALID_PARAM:
1432 nic_hndl, address, length,
1433 dst_cq_hndl, flags, mem_hndl);
1437 nic_hndl, address, length,
1438 dst_cq_hndl, flags, mem_hndl);
1440 /* LBUG never returns, but just for style and consistency */
1448 #define apick_fn "kgnilnd_mem_deregister"
1449 #define apick_fmt "0x%p,0x%p,%d"
1450 static inline gni_return_t kgnilnd_mem_deregister(
1451 IN gni_nic_handle_t nic_hndl,
1452 IN gni_mem_handle_t *mem_hndl,
1458 /* Error injection */
1459 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_UNMAP)) {
1460 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
1462 rrc = gni_mem_deregister(
1463 nic_hndl, mem_hndl, hold_timeout);
1467 case GNI_RC_SUCCESS:
1469 case GNI_RC_INVALID_PARAM:
1471 nic_hndl, mem_hndl, hold_timeout);
1475 nic_hndl, mem_hndl, hold_timeout);
1477 /* LBUG never returns, but just for style and consistency */
1485 #define apick_fn "kgnilnd_mem_mdd_release"
1486 #define apick_fmt "0x%p,0x%p"
1487 static inline gni_return_t kgnilnd_mem_mdd_release(
1488 IN gni_nic_handle_t nic_hndl,
1489 IN gni_mem_handle_t *mem_hndl
1494 /* Error injection */
1495 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_MDD_RELEASE)) {
1496 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NO_MATCH;
1498 rrc = gni_mem_mdd_release(
1499 nic_hndl, mem_hndl);
1503 case GNI_RC_SUCCESS:
1504 case GNI_RC_NO_MATCH:
1508 nic_hndl, mem_hndl);
1510 /* LBUG never returns, but just for style and consistency */
1518 #endif /* _GNILND_API_WRAP_H */