2 * Copyright (C) 2009-2012 Cray, Inc.
4 * Author: Nic Henke <nic@cray.com>
6 * This file is part of Lustre, http://www.lustre.org.
8 * Lustre is free software; you can redistribute it and/or
9 * modify it under the terms of version 2 of the GNU General Public
10 * License as published by the Free Software Foundation.
12 * Lustre is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with Lustre; if not, write to the Free Software
19 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 #ifndef _GNILND_API_WRAP_H
23 #define _GNILND_API_WRAP_H
25 /* LNet is allocated failure locations 0xe000 to 0xffff */
27 /* GNILND has 0xf0XX */
28 #define CFS_FAIL_GNI 0xf000
29 #define CFS_FAIL_GNI_PHYS_MAP 0xf001
30 #define CFS_FAIL_GNI_VIRT_MAP 0xf002
31 #define CFS_FAIL_GNI_GET_UNMAP 0xf003
32 #define CFS_FAIL_GNI_PUT_UNMAP 0xf004
33 #define CFS_FAIL_GNI_MAP_TX 0xf005
34 #define CFS_FAIL_GNI_SMSG_SEND 0xf006
35 #define CFS_FAIL_GNI_CLOSE_SEND 0xf007
36 #define CFS_FAIL_GNI_CDM_CREATE 0xf008
37 #define CFS_FAIL_GNI_CDM_DESTROY 0xf009
38 #define CFS_FAIL_GNI_CDM_ATTACH 0xf00a
39 #define CFS_FAIL_GNI_CQ_CREATE 0xf00b
40 #define CFS_FAIL_GNI_CQ_DESTROY 0xf00c
41 #define CFS_FAIL_GNI_EP_BIND 0xf00d
42 #define CFS_FAIL_GNI_EP_UNBIND 0xf00e
43 #define CFS_FAIL_GNI_EP_SET_EVDATA 0xf00f
44 #define CFS_FAIL_GNI_SMSG_INIT 0xf010
45 #define CFS_FAIL_GNI_SMSG_RELEASE 0xf011
46 #define CFS_FAIL_GNI_POST_RDMA 0xf012
47 #define CFS_FAIL_GNI_GET_COMPLETED 0xf013
48 #define CFS_FAIL_GNI_EP_DESTROY 0xf015
49 #define CFS_FAIL_GNI_VIRT_UNMAP 0xf016
50 #define CFS_FAIL_GNI_MDD_RELEASE 0xf017
51 #define CFS_FAIL_GNI_NOOP_SEND 0xf018
52 #define CFS_FAIL_GNI_ERR_SUBSCRIBE 0xf01a
53 #define CFS_FAIL_GNI_QUIESCE_RACE 0xf01b
54 #define CFS_FAIL_GNI_DG_TERMINATE 0xf01c
55 #define CFS_FAIL_GNI_REG_QUIESCE 0xf01d
56 #define CFS_FAIL_GNI_IN_QUIESCE 0xf01e
57 #define CFS_FAIL_GNI_DELAY_RDMA 0xf01f
58 #define CFS_FAIL_GNI_SR_DOWN_RACE 0xf020
59 #define CFS_FAIL_GNI_ALLOC_TX 0xf021
60 #define CFS_FAIL_GNI_FMABLK_AVAIL 0xf022
61 #define CFS_FAIL_GNI_EP_CREATE 0xf023
62 #define CFS_FAIL_GNI_CQ_GET_EVENT 0xf024
63 #define CFS_FAIL_GNI_PROBE 0xf025
64 #define CFS_FAIL_GNI_EP_TEST 0xf026
65 #define CFS_FAIL_GNI_CONNREQ_DROP 0xf027
66 #define CFS_FAIL_GNI_CONNREQ_PROTO 0xf028
67 #define CFS_FAIL_GNI_CONND_PILEUP 0xf029
68 #define CFS_FAIL_GNI_PHYS_SETUP 0xf02a
69 #define CFS_FAIL_GNI_FIND_TARGET 0xf02b
70 #define CFS_FAIL_GNI_WC_DGRAM_FREE 0xf02c
71 #define CFS_FAIL_GNI_DROP_CLOSING 0xf02d
72 #define CFS_FAIL_GNI_RX_CLOSE_CLOSING 0xf02e
73 #define CFS_FAIL_GNI_RX_CLOSE_CLOSED 0xf02f
74 #define CFS_FAIL_GNI_EP_POST 0xf030
75 #define CFS_FAIL_GNI_PACK_SRCNID 0xf031
76 #define CFS_FAIL_GNI_PACK_DSTNID 0xf032
77 #define CFS_FAIL_GNI_PROBE_WAIT 0xf033
78 #define CFS_FAIL_GNI_SMSG_CKSUM1 0xf034
79 #define CFS_FAIL_GNI_SMSG_CKSUM2 0xf035
80 #define CFS_FAIL_GNI_SMSG_CKSUM3 0xf036
81 #define CFS_FAIL_GNI_DROP_DESTROY_EP 0xf037
82 #define CFS_FAIL_GNI_SMSG_GETNEXT 0xf038
83 #define CFS_FAIL_GNI_FINISH_PURG 0xf039
84 #define CFS_FAIL_GNI_PURG_REL_DELAY 0xf03a
85 #define CFS_FAIL_GNI_DONT_NOTIFY 0xf03b
86 #define CFS_FAIL_GNI_VIRT_SMALL_MAP 0xf03c
87 #define CFS_FAIL_GNI_DELAY_RDMAQ 0xf03d
88 #define CFS_FAIL_GNI_PAUSE_SHUTDOWN 0xf03e
89 #define CFS_FAIL_GNI_PAUSE_DGRAM_COMP 0xf03f
90 #define CFS_FAIL_GNI_NET_LOOKUP 0xf040
91 #define CFS_FAIL_GNI_RECV_TIMEOUT 0xf041
92 #define CFS_FAIL_GNI_SEND_TIMEOUT 0xf042
93 #define CFS_FAIL_GNI_ONLY_NOOP 0xf043
94 #define CFS_FAIL_GNI_FINISH_PURG2 0xf044
95 #define CFS_FAIL_GNI_RACE_RESET 0xf045
96 #define CFS_FAIL_GNI_GNP_CONNECTING1 0xf046
97 #define CFS_FAIL_GNI_GNP_CONNECTING2 0xf047
98 #define CFS_FAIL_GNI_GNP_CONNECTING3 0xf048
99 #define CFS_FAIL_GNI_SCHEDULE_COMPLETE 0xf049
100 #define CFS_FAIL_GNI_PUT_ACK_AGAIN 0xf050
101 #define CFS_FAIL_GNI_GET_REQ_AGAIN 0xf051
102 #define CFS_FAIL_GNI_SCHED_DEADLINE 0xf052
103 #define CFS_FAIL_GNI_DGRAM_DEADLINE 0xf053
104 #define CFS_FAIL_GNI_DGRAM_DROP_TX 0xf054
105 #define CFS_FAIL_GNI_RDMA_CQ_ERROR 0xf055
109 _kgnilnd_api_rc_lbug(const char *rcstr, int rc, struct libcfs_debug_msg_data *data,
110 const char *fmt, ...)
111 __attribute__ ((format (printf, 4, 5)));
113 #define kgnilnd_api_rc_lbug(msgdata, rc, fmt, a...) \
115 /* we don't mask this - it is always at D_ERROR */ \
116 _kgnilnd_api_rc_lbug(kgnilnd_api_rc2str(rc), (rc), msgdata, fmt, ##a); \
119 #define DO_RETCODE(x) case x: return #x;
120 static inline const char *
121 kgnilnd_api_rc2str(gni_return_t rrc)
125 DO_RETCODE(GNI_RC_SUCCESS)
126 DO_RETCODE(GNI_RC_NOT_DONE);
127 DO_RETCODE(GNI_RC_INVALID_PARAM);
128 DO_RETCODE(GNI_RC_ERROR_RESOURCE);
129 DO_RETCODE(GNI_RC_TIMEOUT);
130 DO_RETCODE(GNI_RC_PERMISSION_ERROR);
131 DO_RETCODE(GNI_RC_DESCRIPTOR_ERROR);
132 DO_RETCODE(GNI_RC_ALIGNMENT_ERROR);
133 DO_RETCODE(GNI_RC_INVALID_STATE);
134 DO_RETCODE(GNI_RC_NO_MATCH);
135 DO_RETCODE(GNI_RC_SIZE_ERROR);
136 DO_RETCODE(GNI_RC_TRANSACTION_ERROR);
137 DO_RETCODE(GNI_RC_ILLEGAL_OP);
138 DO_RETCODE(GNI_RC_ERROR_NOMEM);
144 /* log an error and LBUG for unhandled rc from gni api function
145 * the fmt should be something like:
146 * gni_api_call(arg1, arg2, arg3)
149 /* apick_fn and apick_fmt should be defined for each site */
153 #define GNILND_API_RC_LBUG(args...) \
155 LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL); \
156 kgnilnd_api_rc_lbug(&msgdata, rrc, apick_fn"("apick_fmt")", ##args); \
159 #define GNILND_API_SWBUG(args...) \
161 CERROR("likely SOFTWARE BUG "apick_fn"("apick_fmt") rc %s\n", \
162 ##args, kgnilnd_api_rc2str(rrc)); \
165 #define GNILND_API_EINVAL(args...) \
167 CERROR("invalid parameter to "apick_fn"("apick_fmt") rc %s\n", \
168 ##args, kgnilnd_api_rc2str(rrc)); \
171 #define GNILND_API_RESOURCE(args...) \
173 CERROR("no resources for "apick_fn"("apick_fmt") rc %s\n", \
174 ##args, kgnilnd_api_rc2str(rrc)); \
177 #define GNILND_API_BUSY(args...) \
179 CERROR("resources busy for "apick_fn"("apick_fmt") rc %s\n", \
180 ##args, kgnilnd_api_rc2str(rrc)); \
183 #undef DEBUG_SMSG_CREDITS
184 #ifdef DEBUG_SMSG_CREDITS
185 #define CRAY_CONFIG_GHAL_GEMINI
186 #include <gni_priv.h>
187 #define GNIDBG_SMSG_CREDS(level, conn) \
189 gni_ep_smsg_mbox_t *smsg = conn->gnc_ephandle->smsg; \
190 CDEBUG(level, "SMSGDBG: conn %p mcred %d/%d bcred %d/%d " \
191 "s_seq %d/%d/%d r_seq %d/%d/%d retr %d\n", \
192 conn, smsg->mbox_credits, smsg->back_mbox_credits, \
193 smsg->buffer_credits, smsg->back_buffer_credits, \
194 smsg->s_seqno, smsg->s_seqno_back_mbox_credits, \
195 smsg->s_seqno_back_buffer_credits, smsg->r_seqno, \
196 smsg->r_seqno_back_mbox_credits, \
197 smsg->r_seqno_back_buffer_credits, smsg->retransmit_count); \
200 #define GNIDBG_SMSG_CREDS(level, conn) do {} while(0)
203 /* these are all wrappers around gni_XXX functions.
204 * This allows us to handle all the return codes and api checks without
205 * dirtying up the logic code */
207 /* TODO: RETURN wrapper that translates integer to GNI API RC string */
209 #define apick_fn "kgnilnd_cdm_create"
210 #define apick_fmt "%u, %u, %u, %u, 0x%p"
211 static inline gni_return_t kgnilnd_cdm_create(
216 OUT gni_cdm_handle_t *cdm_hndl
221 /* Error injection */
222 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CDM_CREATE)) {
223 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
225 rrc = gni_cdm_create(inst_id, ptag, cookie, modes, cdm_hndl);
231 case GNI_RC_ERROR_RESOURCE:
232 case GNI_RC_INVALID_PARAM:
233 /* Try to bail gracefully */
235 inst_id, ptag, cookie, modes, cdm_hndl);
239 inst_id, ptag, cookie, modes, cdm_hndl);
241 /* LBUG never returns, but just for style and consistency */
250 #define apick_fn "kgnilnd_cdm_attach"
251 #define apick_fmt "0x%p, %u, 0x%p, 0x%p"
252 static inline gni_return_t kgnilnd_cdm_attach(
253 IN gni_cdm_handle_t cdm_hndl,
254 IN uint32_t device_id,
255 OUT uint32_t *local_addr,
256 OUT gni_nic_handle_t *nic_hndl
261 /* Error injection */
262 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CDM_ATTACH)) {
263 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
265 rrc = gni_cdm_attach(cdm_hndl, device_id, local_addr, nic_hndl);
271 case GNI_RC_NO_MATCH:
272 case GNI_RC_INVALID_PARAM:
274 cdm_hndl, device_id, local_addr, nic_hndl);
276 case GNI_RC_ERROR_RESOURCE:
277 case GNI_RC_INVALID_STATE:
279 cdm_hndl, device_id, local_addr, nic_hndl);
283 cdm_hndl, device_id, local_addr, nic_hndl);
285 /* LBUG never returns, but just for style and consistency */
293 #define apick_fn "kgnilnd_cdm_destroy"
294 #define apick_fmt "0x%p"
295 static inline gni_return_t kgnilnd_cdm_destroy(
296 IN gni_cdm_handle_t cdm_hndl
301 /* Error injection */
302 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_DESTROY)) {
303 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
305 rrc = gni_cdm_destroy(
312 case GNI_RC_INVALID_PARAM:
320 /* LBUG never returns, but just for style and consistency */
328 #define apick_fn "kgnilnd_subscribe_errors"
329 #define apick_fmt "0x%p,%x,%u,0x%p,0x%p,0x%p"
330 static inline gni_return_t kgnilnd_subscribe_errors(
331 IN gni_nic_handle_t nic_handle,
332 IN gni_error_mask_t mask,
333 IN uint32_t EEQ_size,
334 IN void (*EQ_new_event)(gni_err_handle_t),
335 IN void (*app_crit_err)(gni_err_handle_t),
336 OUT gni_err_handle_t *err_handle
341 /* Error injection */
342 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_ERR_SUBSCRIBE)) {
343 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
345 rrc = gni_subscribe_errors(
346 nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
353 case GNI_RC_INVALID_PARAM:
355 nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
358 case GNI_RC_ERROR_RESOURCE:
360 nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
365 nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
367 /* LBUG never returns, but just for style and consistency */
375 #define apick_fn "kgnilnd_release_errors"
376 #define apick_fmt "0x%p"
377 static inline gni_return_t kgnilnd_release_errors(
378 IN gni_err_handle_t err_handle
383 rrc = gni_release_errors(
389 case GNI_RC_INVALID_PARAM:
390 case GNI_RC_NOT_DONE:
397 /* LBUG never returns, but just for style and consistency */
405 #define apick_fn "kgnilnd_set_quiesce_callback"
406 #define apick_fmt "0x%p,0x%p"
407 static inline gni_return_t kgnilnd_set_quiesce_callback(
408 IN gni_nic_handle_t nic_handle,
409 IN void (*qsce_func)(gni_nic_handle_t, uint64_t msecs)
414 /* Error injection */
415 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_REG_QUIESCE)) {
416 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
418 rrc = gni_set_quiesce_callback(
419 nic_handle, qsce_func);
425 case GNI_RC_INVALID_STATE:
426 case GNI_RC_INVALID_PARAM:
428 nic_handle, qsce_func);
432 nic_handle, qsce_func);
433 /* LBUG never returns, but just for style and consistency */
441 #define apick_fn "kgnilnd_get_quiesce_status"
442 #define apick_fmt "0x%p"
443 static inline gni_return_t kgnilnd_get_quiesce_status(
444 IN gni_nic_handle_t nic_handle
449 /* this has weird RC -
450 * 0 - quiesce not in progress
451 * 1 - quiesce is turned on
454 /* Error injection */
455 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_IN_QUIESCE)) {
458 rrc = gni_get_quiesce_status(
469 /* LBUG never returns, but just for style and consistency */
477 #define apick_fn "kgnilnd_cq_create"
478 #define apick_fmt "0x%p, %u, %u, 0x%p, %#llx, 0x%p"
479 static inline gni_return_t kgnilnd_cq_create(
480 IN gni_nic_handle_t nic_hndl,
481 IN uint32_t entry_count,
482 IN uint32_t delay_index,
483 IN gni_cq_event_hndlr_f *event_handler,
484 IN uint64_t usr_event_data,
485 OUT gni_cq_handle_t *cq_hndl
490 /* Error injection */
491 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_CREATE)) {
492 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
495 nic_hndl, entry_count, delay_index, event_handler,
496 usr_event_data, cq_hndl);
502 case GNI_RC_INVALID_PARAM:
504 nic_hndl, entry_count, delay_index, event_handler,
505 usr_event_data, cq_hndl);
507 case GNI_RC_ERROR_RESOURCE:
509 nic_hndl, entry_count, delay_index, event_handler,
510 usr_event_data, cq_hndl);
514 nic_hndl, entry_count, delay_index, event_handler,
515 usr_event_data, cq_hndl);
517 /* LBUG never returns, but just for style and consistency */
525 #define apick_fn "kgnilnd_cq_destroy"
526 #define apick_fmt "0x%p"
527 static inline gni_return_t kgnilnd_cq_destroy(
528 IN gni_cq_handle_t cq_hndl
533 /* Error injection */
534 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_DESTROY)) {
535 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
538 rrc = gni_cq_destroy(
545 case GNI_RC_INVALID_PARAM:
549 case GNI_RC_ERROR_RESOURCE:
557 /* LBUG never returns, but just for style and consistency */
565 #define apick_fn "kgnilnd_cq_get_event"
566 #define apick_fmt "0x%p, 0x%p"
567 static inline gni_return_t kgnilnd_cq_get_event(
568 IN gni_cq_handle_t cq_hndl,
569 OUT gni_cq_entry_t *event_data
574 /* no error injection - CQs are touchy about the data.
575 * where appropriate, we'll do this on the CQs that should be able to
576 * handle the various errors */
577 rrc = gni_cq_get_event(
578 cq_hndl, event_data);
582 case GNI_RC_NOT_DONE:
583 case GNI_RC_TRANSACTION_ERROR:
585 case GNI_RC_ERROR_RESOURCE:
586 LASSERTF(GNI_CQ_OVERRUN(*event_data),
587 "kgni returned ERROR_RESOURCE but cq_hndl 0x%p is not "
588 "overrun\n", cq_hndl);
590 case GNI_RC_INVALID_PARAM:
592 cq_hndl, event_data);
596 cq_hndl, event_data);
598 /* LBUG never returns, but just for style and consistency */
606 #define apick_fn "kgnilnd_smsg_init"
607 #define apick_fmt "0x%p, 0x%p, 0x%p"
608 static inline gni_return_t kgnilnd_smsg_init(
609 IN gni_ep_handle_t ep_hndl,
610 IN gni_smsg_attr_t *local_smsg_attr,
611 IN gni_smsg_attr_t *remote_smsg_attr
616 /* Error injection */
617 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_INIT)) {
618 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_RESOURCE;
621 ep_hndl, local_smsg_attr, remote_smsg_attr);
625 /* both of these are OK, upper SW needs to handle */
627 case GNI_RC_NOT_DONE:
629 case GNI_RC_INVALID_PARAM:
630 case GNI_RC_INVALID_STATE:
632 ep_hndl, local_smsg_attr, remote_smsg_attr);
634 case GNI_RC_ERROR_RESOURCE:
636 ep_hndl, local_smsg_attr, remote_smsg_attr);
640 ep_hndl, local_smsg_attr, remote_smsg_attr);
642 /* LBUG never returns, but just for style and consistency */
650 #define apick_fn "kgnilnd_smsg_send"
651 #define apick_fmt "0x%p, 0x%p, %d, 0x%p, %u %u"
652 static inline gni_return_t kgnilnd_smsg_send(
653 IN gni_ep_handle_t ep_hndl,
655 IN uint32_t header_length,
657 IN uint32_t data_length,
663 /* Error injection */
664 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_SEND)) {
665 if (cfs_fail_loc & CFS_FAIL_RAND) {
666 rrc = GNI_RC_NOT_DONE;
668 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
672 ep_hndl, header, header_length, data, data_length, msg_id);
676 /* both of these are OK, upper SW needs to handle */
678 case GNI_RC_NOT_DONE:
680 case GNI_RC_INVALID_PARAM:
682 ep_hndl, header, header_length, data, data_length, msg_id);
684 case GNI_RC_ERROR_RESOURCE:
686 ep_hndl, header, header_length, data, data_length, msg_id);
690 ep_hndl, header, header_length, data, data_length, msg_id);
692 /* LBUG never returns, but just for style and consistency */
700 #define apick_fn "kgnilnd_smsg_getnext"
701 #define apick_fmt "0x%p,0x%p"
702 static inline gni_return_t kgnilnd_smsg_getnext(
703 IN gni_ep_handle_t ep_hndl,
709 /* Error injection */
710 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_RELEASE)) {
711 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
713 rrc = gni_smsg_getnext(
718 /* both of these are OK, upper SW needs to handle */
720 case GNI_RC_NOT_DONE:
721 case GNI_RC_INVALID_STATE:
723 case GNI_RC_INVALID_PARAM:
731 /* LBUG never returns, but just for style and consistency */
739 #define apick_fn "kgnilnd_smsg_release"
740 #define apick_fmt "0x%p"
741 static inline gni_return_t kgnilnd_smsg_release(
742 IN gni_ep_handle_t ep_hndl
747 /* Error injection */
748 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_RELEASE)) {
749 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
751 rrc = gni_smsg_release(
756 /* both of these are OK, upper SW needs to handle */
758 case GNI_RC_NOT_DONE:
760 case GNI_RC_INVALID_PARAM:
768 /* LBUG never returns, but just for style and consistency */
776 #define apick_fn "kgnilnd_ep_create"
777 #define apick_fmt "0x%p, 0x%p, 0x%p"
778 static inline gni_return_t kgnilnd_ep_create(
779 IN gni_nic_handle_t nic_hndl,
780 IN gni_cq_handle_t src_cq_hndl,
781 OUT gni_ep_handle_t *ep_hndl
786 /* error injection */
787 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_CREATE)) {
788 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_NOMEM;
791 nic_hndl, src_cq_hndl, ep_hndl);
797 case GNI_RC_INVALID_PARAM:
799 nic_hndl, src_cq_hndl, ep_hndl);
801 case GNI_RC_ERROR_NOMEM:
803 nic_hndl, src_cq_hndl, ep_hndl);
807 nic_hndl, src_cq_hndl, ep_hndl);
809 /* lbug never returns, but just for style and consistency */
817 #define apick_fn "kgnilnd_ep_bind"
818 #define apick_fmt "0x%p, %x, %x"
819 static inline gni_return_t kgnilnd_ep_bind(
820 IN gni_ep_handle_t ep_hndl,
821 IN uint32_t remote_addr,
822 IN uint32_t remote_id
827 /* error injection */
828 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_BIND)) {
829 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
832 ep_hndl, remote_addr, remote_id);
836 /* both of these are ok, upper sw needs to handle */
838 case GNI_RC_NOT_DONE:
840 case GNI_RC_INVALID_PARAM:
842 ep_hndl, remote_addr, remote_id);
846 ep_hndl, remote_addr, remote_id);
848 /* lbug never returns, but just for style and consistency */
856 #define apick_fn "kgnilnd_ep_set_eventdata"
857 #define apick_fmt "0x%p, %x, %x"
858 static inline gni_return_t kgnilnd_ep_set_eventdata(
859 IN gni_ep_handle_t ep_hndl,
860 IN uint32_t local_event,
861 IN uint32_t remote_event
866 /* Error injection */
867 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_SET_EVDATA)) {
868 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
870 rrc = gni_ep_set_eventdata(
871 ep_hndl, local_event, remote_event);
877 case GNI_RC_INVALID_PARAM:
879 ep_hndl, local_event, remote_event);
883 ep_hndl, local_event, remote_event);
885 /* LBUG never returns, but just for style and consistency */
893 #define apick_fn "kgnilnd_ep_unbind"
894 #define apick_fmt "0x%p"
895 static inline gni_return_t kgnilnd_ep_unbind(
896 IN gni_ep_handle_t ep_hndl
901 /* Error injection */
902 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_UNBIND)) {
903 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
910 /* both of these are OK, upper SW needs to handle */
911 case GNI_RC_NOT_DONE:
914 case GNI_RC_INVALID_PARAM:
922 /* LBUG never returns, but just for style and consistency */
930 #define apick_fn "kgnilnd_ep_destroy"
931 #define apick_fmt "0x%p"
932 static inline gni_return_t kgnilnd_ep_destroy(
933 IN gni_ep_handle_t ep_hndl
938 /* Error injection */
939 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_DESTROY)) {
940 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
942 rrc = gni_ep_destroy(
949 case GNI_RC_INVALID_PARAM:
957 /* LBUG never returns, but just for style and consistency */
965 #define apick_fn "kgnilnd_ep_postdata_w_id"
966 #define apick_fmt "0x%p, 0x%p, %d, 0x%p, %d, %llu"
967 static inline gni_return_t kgnilnd_ep_postdata_w_id(
968 IN gni_ep_handle_t ep_hndl,
970 IN uint16_t data_len,
972 IN uint16_t buf_size,
973 IN uint64_t datagram_id
978 /* Error injection */
979 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_POST)) {
980 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_SIZE_ERROR;
982 rrc = gni_ep_postdata_w_id(
983 ep_hndl, in_data, data_len, out_buf, buf_size,
989 case GNI_RC_ERROR_NOMEM:
990 case GNI_RC_ERROR_RESOURCE:
992 case GNI_RC_INVALID_PARAM:
993 case GNI_RC_SIZE_ERROR:
995 ep_hndl, in_data, data_len, out_buf, buf_size,
1000 ep_hndl, in_data, data_len, out_buf, buf_size,
1003 /* LBUG never returns, but just for style and consistency */
1011 #define apick_fn "kgnilnd_ep_postdata_test_by_id"
1012 #define apick_fmt "0x%p, %llu, 0x%p, 0x%p, 0x%p"
1013 static inline gni_return_t kgnilnd_ep_postdata_test_by_id(
1014 IN gni_ep_handle_t ep_hndl,
1015 IN uint64_t datagram_id,
1016 OUT gni_post_state_t *post_state,
1017 OUT uint32_t *remote_addr,
1018 OUT uint32_t *remote_id
1023 /* Error injection */
1024 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_TEST)) {
1025 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_NOMEM;
1027 rrc = gni_ep_postdata_test_by_id(
1028 ep_hndl, datagram_id, post_state, remote_addr,
1031 /* we want to lie, but we need to do the actual work first
1032 * so we don't keep getting the event saying a dgram is ready */
1033 if (rrc == GNI_RC_SUCCESS && CFS_FAIL_CHECK(CFS_FAIL_GNI_DG_TERMINATE)) {
1034 /* don't use fail_val, allows us to do FAIL_SOME */
1035 *post_state = GNI_POST_TERMINATED;
1040 case GNI_RC_SUCCESS:
1041 case GNI_RC_NO_MATCH:
1043 case GNI_RC_SIZE_ERROR:
1044 case GNI_RC_INVALID_PARAM:
1046 ep_hndl, datagram_id, post_state, remote_addr,
1049 case GNI_RC_ERROR_NOMEM:
1050 GNILND_API_RESOURCE(
1051 ep_hndl, datagram_id, post_state, remote_addr,
1056 ep_hndl, datagram_id, post_state, remote_addr,
1059 /* LBUG never returns, but just for style and consistency */
1067 #define apick_fn "kgnilnd_ep_postdata_cancel_by_id"
1068 #define apick_fmt "0x%p, %llu"
1069 static inline gni_return_t kgnilnd_ep_postdata_cancel_by_id(
1070 IN gni_ep_handle_t ep_hndl,
1071 IN uint64_t datagram_id
1076 /* no error injection as the only thing we'd do is LBUG */
1078 rrc = gni_ep_postdata_cancel_by_id(
1079 ep_hndl, datagram_id);
1082 case GNI_RC_SUCCESS:
1083 case GNI_RC_NO_MATCH:
1085 case GNI_RC_INVALID_PARAM:
1087 ep_hndl, datagram_id);
1091 ep_hndl, datagram_id);
1093 /* LBUG never returns, but just for style and consistency */
1101 #define apick_fn "kgnilnd_postdata_probe_by_id"
1102 #define apick_fmt "0x%p, 0x%p"
1103 static inline gni_return_t kgnilnd_postdata_probe_by_id(
1104 IN gni_nic_handle_t nic_hndl,
1105 OUT uint64_t *datagram_id
1110 /* Error injection */
1111 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PROBE)) {
1112 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NO_MATCH;
1114 rrc = gni_postdata_probe_by_id(
1115 nic_hndl, datagram_id);
1119 case GNI_RC_SUCCESS:
1120 case GNI_RC_NO_MATCH:
1122 case GNI_RC_INVALID_PARAM:
1124 nic_hndl, datagram_id);
1128 nic_hndl, datagram_id);
1130 /* LBUG never returns, but just for style and consistency */
1138 #define apick_fn "kgnilnd_postdata_probe_wait_by_id"
1139 #define apick_fmt "0x%p, %d, 0x%p"
1140 static inline gni_return_t kgnilnd_postdata_probe_wait_by_id(
1141 IN gni_nic_handle_t nic_hndl,
1142 IN uint32_t timeout,
1143 OUT uint64_t *datagram_id
1148 /* Error injection */
1149 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PROBE_WAIT)) {
1150 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_TIMEOUT;
1152 rrc = gni_postdata_probe_wait_by_id(
1153 nic_hndl, timeout, datagram_id);
1157 case GNI_RC_SUCCESS:
1158 case GNI_RC_TIMEOUT:
1160 case GNI_RC_INVALID_PARAM:
1162 nic_hndl, timeout, datagram_id);
1166 nic_hndl, timeout, datagram_id);
1168 /* LBUG never returns, but just for style and consistency */
1176 #define apick_fn "kgnilnd_post_rdma"
1177 #define apick_fmt "0x%p, 0x%p"
1178 static inline gni_return_t kgnilnd_post_rdma(
1179 IN gni_ep_handle_t ep_hndl,
1180 IN gni_post_descriptor_t *post_descr
1185 /* Error injection */
1186 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_POST_RDMA)) {
1187 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
1189 rrc = gni_post_rdma(
1190 ep_hndl, post_descr);
1194 case GNI_RC_SUCCESS:
1196 case GNI_RC_ALIGNMENT_ERROR:
1197 case GNI_RC_INVALID_PARAM:
1199 ep_hndl, post_descr);
1201 case GNI_RC_ERROR_RESOURCE:
1202 CDEBUG(D_NET, "no resources for kgnilnd_post_rdma (0x%p, 0x%p)"
1203 " rc %s\n", ep_hndl, post_descr,
1204 kgnilnd_api_rc2str(rrc));
1208 ep_hndl, post_descr);
1210 /* LBUG never returns, but just for style and consistency */
1218 #define apick_fn "kgnilnd_get_completed"
1219 #define apick_fmt "0x%p,%#llx,0x%p"
1220 static inline gni_return_t kgnilnd_get_completed(
1221 IN gni_cq_handle_t cq_hndl,
1222 IN gni_cq_entry_t event_data,
1223 OUT gni_post_descriptor_t **post_descr
1229 rrc = gni_get_completed(cq_hndl, event_data, post_descr);
1232 case GNI_RC_TRANSACTION_ERROR:
1233 case GNI_RC_SUCCESS:
1235 case GNI_RC_DESCRIPTOR_ERROR:
1236 case GNI_RC_INVALID_PARAM:
1237 GNILND_API_SWBUG(cq_hndl, event_data, post_descr);
1240 GNILND_API_RC_LBUG(cq_hndl, event_data, post_descr);
1241 /* LBUG never returns, but just for style and consistency */
1245 /* Error injection - we need a valid desc, so let kgni give us one
1247 if (rrc == GNI_RC_SUCCESS &&
1248 (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED))) {
1249 /* We only trigger TRANSACTION_ERROR for now */
1250 gni_post_descriptor_t *desc;
1251 rrc = GNI_RC_TRANSACTION_ERROR;
1254 /* recoverable decision made from cfs_fail_val in
1255 * kgnilnd_cq_error_str and
1256 * kgnilnd_cq_error_recoverable */
1263 #define apick_fn "kgnilnd_cq_error_str"
1264 #define apick_fmt "%#llx,0x%p,%d"
1265 static inline gni_return_t kgnilnd_cq_error_str(
1266 IN gni_cq_entry_t entry,
1273 /* Error injection - set string if we injected a
1274 * TRANSACTION_ERROR earlier */
1275 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED)) {
1276 /* if we just set persistent error, we can't ever
1277 * break in via ssh to clear, so use a count > 10 to indicate fatal */
1278 sprintf(buffer, "INJECT:%s", cfs_fail_val > 10 ?
1279 "FATAL" : "RECOVERABLE");
1280 rrc = GNI_RC_SUCCESS;
1282 rrc = gni_cq_error_str(
1283 entry, buffer, len);
1287 case GNI_RC_SUCCESS:
1289 case GNI_RC_SIZE_ERROR:
1290 case GNI_RC_INVALID_PARAM:
1292 entry, buffer, len);
1293 /* give them something to use */
1294 snprintf(buffer, len, "UNDEF:UNDEF");
1298 entry, buffer, len);
1300 /* LBUG never returns, but just for style and consistency */
1308 #define apick_fn "kgnilnd_cq_error_recoverable"
1309 #define apick_fmt "%#llx,0x%p"
1310 static inline gni_return_t kgnilnd_cq_error_recoverable(
1311 IN gni_cq_entry_t entry,
1312 IN uint32_t *recoverable
1317 /* Error injection - set string if we injected a
1318 * TRANSACTION_ERROR earlier */
1319 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED)) {
1320 *recoverable = cfs_fail_val > 10 ? 0 : 1;
1321 rrc = GNI_RC_SUCCESS;
1323 rrc = gni_cq_error_recoverable(
1324 entry, recoverable);
1328 case GNI_RC_SUCCESS:
1330 case GNI_RC_INVALID_STATE:
1331 case GNI_RC_INVALID_PARAM:
1333 entry, recoverable);
1338 entry, recoverable);
1340 /* LBUG never returns, but just for style and consistency */
1348 #define apick_fn "kgnilnd_mem_register_segments"
1349 #define apick_fmt "0x%p,0x%p,%u,0x%p,%x,0x%p"
1350 static inline gni_return_t
1351 kgnilnd_mem_register_segments(
1352 IN gni_nic_handle_t nic_hndl,
1353 IN gni_mem_segment_t *mem_segments,
1354 IN uint32_t segments_cnt,
1355 IN gni_cq_handle_t dst_cq_hndl,
1357 OUT gni_mem_handle_t *mem_hndl
1362 /* Error injection */
1363 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PHYS_MAP)) {
1364 rrc = GNI_RC_ERROR_RESOURCE;
1366 rrc = gni_mem_register_segments(
1367 nic_hndl, mem_segments, segments_cnt,
1368 dst_cq_hndl, flags, mem_hndl);
1372 case GNI_RC_SUCCESS:
1373 case GNI_RC_ERROR_RESOURCE:
1375 case GNI_RC_INVALID_PARAM:
1377 nic_hndl, mem_segments, segments_cnt,
1378 dst_cq_hndl, flags, mem_hndl);
1382 nic_hndl, mem_segments, segments_cnt,
1383 dst_cq_hndl, flags, mem_hndl);
1385 /* LBUG never returns, but just for style and consistency */
1393 #define apick_fn "kgnilnd_mem_register"
1394 #define apick_fmt "0x%p,%#llx,%#llx0x%p,%u,0x%p"
1395 static inline gni_return_t kgnilnd_mem_register(
1396 IN gni_nic_handle_t nic_hndl,
1397 IN uint64_t address,
1399 IN gni_cq_handle_t dst_cq_hndl,
1401 OUT gni_mem_handle_t *mem_hndl
1406 /* Error injection */
1407 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_MAP)) {
1408 rrc = GNI_RC_ERROR_RESOURCE;
1409 } else if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_SMALL_MAP) &&
1410 length <= *kgnilnd_tunables.kgn_max_immediate) {
1411 rrc = GNI_RC_INVALID_PARAM;
1413 rrc = gni_mem_register(
1414 nic_hndl, address, length,
1415 dst_cq_hndl, flags, mem_hndl);
1418 /* gni_mem_register may return GNI_RC_ERROR_NOMEM under memory
1419 * pressure but the upper layers only know about resource errors
1421 if (rrc == GNI_RC_ERROR_NOMEM) {
1422 rrc = GNI_RC_ERROR_RESOURCE;
1426 case GNI_RC_SUCCESS:
1427 case GNI_RC_ERROR_RESOURCE:
1429 case GNI_RC_INVALID_PARAM:
1431 nic_hndl, address, length,
1432 dst_cq_hndl, flags, mem_hndl);
1436 nic_hndl, address, length,
1437 dst_cq_hndl, flags, mem_hndl);
1439 /* LBUG never returns, but just for style and consistency */
1447 #define apick_fn "kgnilnd_mem_deregister"
1448 #define apick_fmt "0x%p,0x%p,%d"
1449 static inline gni_return_t kgnilnd_mem_deregister(
1450 IN gni_nic_handle_t nic_hndl,
1451 IN gni_mem_handle_t *mem_hndl,
1457 /* Error injection */
1458 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_UNMAP)) {
1459 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
1461 rrc = gni_mem_deregister(
1462 nic_hndl, mem_hndl, hold_timeout);
1466 case GNI_RC_SUCCESS:
1468 case GNI_RC_INVALID_PARAM:
1470 nic_hndl, mem_hndl, hold_timeout);
1474 nic_hndl, mem_hndl, hold_timeout);
1476 /* LBUG never returns, but just for style and consistency */
1484 #define apick_fn "kgnilnd_mem_mdd_release"
1485 #define apick_fmt "0x%p,0x%p"
1486 static inline gni_return_t kgnilnd_mem_mdd_release(
1487 IN gni_nic_handle_t nic_hndl,
1488 IN gni_mem_handle_t *mem_hndl
1493 /* Error injection */
1494 if (CFS_FAIL_CHECK(CFS_FAIL_GNI_MDD_RELEASE)) {
1495 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NO_MATCH;
1497 rrc = gni_mem_mdd_release(
1498 nic_hndl, mem_hndl);
1502 case GNI_RC_SUCCESS:
1503 case GNI_RC_NO_MATCH:
1507 nic_hndl, mem_hndl);
1509 /* LBUG never returns, but just for style and consistency */
1517 #endif /* _GNILND_API_WRAP_H */