Whamcloud - gitweb
LU-6261 gnilnd: kgnilnd_check_rdma_cq race in error path.
[fs/lustre-release.git] / lnet / klnds / gnilnd / gnilnd_api_wrap.h
1 /*
2  * Copyright (C) 2009-2012 Cray, Inc.
3  *
4  *   Author: Nic Henke <nic@cray.com>
5  *
6  *   This file is part of Lustre, http://www.lustre.org.
7  *
8  *   Lustre is free software; you can redistribute it and/or
9  *   modify it under the terms of version 2 of the GNU General Public
10  *   License as published by the Free Software Foundation.
11  *
12  *   Lustre is distributed in the hope that it will be useful,
13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  *   GNU General Public License for more details.
16  *
17  *   You should have received a copy of the GNU General Public License
18  *   along with Lustre; if not, write to the Free Software
19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20  *
21  */
22 #ifndef _GNILND_API_WRAP_H
23 #define _GNILND_API_WRAP_H
24
25 /* LNet is allocated failure locations 0xe000 to 0xffff */
26
27 /* GNILND has 0xf0XX */
28 #define CFS_FAIL_GNI                    0xf000
29 #define CFS_FAIL_GNI_PHYS_MAP           0xf001
30 #define CFS_FAIL_GNI_VIRT_MAP           0xf002
31 #define CFS_FAIL_GNI_GET_UNMAP          0xf003
32 #define CFS_FAIL_GNI_PUT_UNMAP          0xf004
33 #define CFS_FAIL_GNI_MAP_TX             0xf005
34 #define CFS_FAIL_GNI_SMSG_SEND          0xf006
35 #define CFS_FAIL_GNI_CLOSE_SEND         0xf007
36 #define CFS_FAIL_GNI_CDM_CREATE         0xf008
37 #define CFS_FAIL_GNI_CDM_DESTROY        0xf009
38 #define CFS_FAIL_GNI_CDM_ATTACH         0xf00a
39 #define CFS_FAIL_GNI_CQ_CREATE          0xf00b
40 #define CFS_FAIL_GNI_CQ_DESTROY         0xf00c
41 #define CFS_FAIL_GNI_EP_BIND            0xf00d
42 #define CFS_FAIL_GNI_EP_UNBIND          0xf00e
43 #define CFS_FAIL_GNI_EP_SET_EVDATA      0xf00f
44 #define CFS_FAIL_GNI_SMSG_INIT          0xf010
45 #define CFS_FAIL_GNI_SMSG_RELEASE       0xf011
46 #define CFS_FAIL_GNI_POST_RDMA          0xf012
47 #define CFS_FAIL_GNI_GET_COMPLETED      0xf013
48 #define CFS_FAIL_GNI_EP_DESTROY         0xf015
49 #define CFS_FAIL_GNI_VIRT_UNMAP         0xf016
50 #define CFS_FAIL_GNI_MDD_RELEASE        0xf017
51 #define CFS_FAIL_GNI_NOOP_SEND          0xf018
52 #define CFS_FAIL_GNI_ERR_SUBSCRIBE      0xf01a
53 #define CFS_FAIL_GNI_QUIESCE_RACE       0xf01b
54 #define CFS_FAIL_GNI_DG_TERMINATE       0xf01c
55 #define CFS_FAIL_GNI_REG_QUIESCE        0xf01d
56 #define CFS_FAIL_GNI_IN_QUIESCE         0xf01e
57 #define CFS_FAIL_GNI_DELAY_RDMA         0xf01f
58 #define CFS_FAIL_GNI_SR_DOWN_RACE       0xf020
59 #define CFS_FAIL_GNI_ALLOC_TX           0xf021
60 #define CFS_FAIL_GNI_FMABLK_AVAIL       0xf022
61 #define CFS_FAIL_GNI_EP_CREATE          0xf023
62 #define CFS_FAIL_GNI_CQ_GET_EVENT       0xf024
63 #define CFS_FAIL_GNI_PROBE              0xf025
64 #define CFS_FAIL_GNI_EP_TEST            0xf026
65 #define CFS_FAIL_GNI_CONNREQ_DROP       0xf027
66 #define CFS_FAIL_GNI_CONNREQ_PROTO      0xf028
67 #define CFS_FAIL_GNI_CONND_PILEUP       0xf029
68 #define CFS_FAIL_GNI_PHYS_SETUP         0xf02a
69 #define CFS_FAIL_GNI_FIND_TARGET        0xf02b
70 #define CFS_FAIL_GNI_WC_DGRAM_FREE      0xf02c
71 #define CFS_FAIL_GNI_DROP_CLOSING       0xf02d
72 #define CFS_FAIL_GNI_RX_CLOSE_CLOSING   0xf02e
73 #define CFS_FAIL_GNI_RX_CLOSE_CLOSED    0xf02f
74 #define CFS_FAIL_GNI_EP_POST            0xf030
75 #define CFS_FAIL_GNI_PACK_SRCNID        0xf031
76 #define CFS_FAIL_GNI_PACK_DSTNID        0xf032
77 #define CFS_FAIL_GNI_PROBE_WAIT         0xf033
78 #define CFS_FAIL_GNI_SMSG_CKSUM1        0xf034
79 #define CFS_FAIL_GNI_SMSG_CKSUM2        0xf035
80 #define CFS_FAIL_GNI_SMSG_CKSUM3        0xf036
81 #define CFS_FAIL_GNI_DROP_DESTROY_EP    0xf037
82 #define CFS_FAIL_GNI_SMSG_GETNEXT       0xf038
83 #define CFS_FAIL_GNI_FINISH_PURG        0xf039
84 #define CFS_FAIL_GNI_PURG_REL_DELAY     0xf03a
85 #define CFS_FAIL_GNI_DONT_NOTIFY        0xf03b
86 #define CFS_FAIL_GNI_VIRT_SMALL_MAP     0xf03c
87 #define CFS_FAIL_GNI_DELAY_RDMAQ        0xf03d
88 #define CFS_FAIL_GNI_PAUSE_SHUTDOWN     0xf03e
89 #define CFS_FAIL_GNI_PAUSE_DGRAM_COMP   0xf03f
90 #define CFS_FAIL_GNI_NET_LOOKUP         0xf040
91 #define CFS_FAIL_GNI_RECV_TIMEOUT       0xf041
92 #define CFS_FAIL_GNI_SEND_TIMEOUT       0xf042
93 #define CFS_FAIL_GNI_ONLY_NOOP          0xf043
94 #define CFS_FAIL_GNI_FINISH_PURG2       0xf044
95 #define CFS_FAIL_GNI_RACE_RESET         0xf045
96 #define CFS_FAIL_GNI_GNP_CONNECTING1    0xf046
97 #define CFS_FAIL_GNI_GNP_CONNECTING2    0xf047
98 #define CFS_FAIL_GNI_GNP_CONNECTING3    0xf048
99 #define CFS_FAIL_GNI_SCHEDULE_COMPLETE  0xf049
100 #define CFS_FAIL_GNI_PUT_ACK_AGAIN      0xf050
101 #define CFS_FAIL_GNI_GET_REQ_AGAIN      0xf051
102 #define CFS_FAIL_GNI_SCHED_DEADLINE     0xf052
103 #define CFS_FAIL_GNI_DGRAM_DEADLINE     0xf053
104 #define CFS_FAIL_GNI_DGRAM_DROP_TX      0xf054
105 #define CFS_FAIL_GNI_RDMA_CQ_ERROR      0xf055
106
107 /* helper macros */
108 extern void
109 _kgnilnd_api_rc_lbug(const char *rcstr, int rc, struct libcfs_debug_msg_data *data,
110                         const char *fmt, ...)
111         __attribute__ ((format (printf, 4, 5)));
112
113 #define kgnilnd_api_rc_lbug(msgdata, rc, fmt, a...)                             \
114 do {                                                                            \
115         CFS_CHECK_STACK(msgdata, D_ERROR, NULL);                                \
116         /* we don't mask this - it is always at D_ERROR */                      \
117         _kgnilnd_api_rc_lbug(kgnilnd_api_rc2str(rc), (rc), msgdata, fmt, ##a);  \
118 } while (0)
119
120 #define DO_RETCODE(x) case x: return #x;
121 static inline const char *
122 kgnilnd_api_rc2str(gni_return_t rrc)
123 {
124
125         switch (rrc) {
126                 DO_RETCODE(GNI_RC_SUCCESS)
127                 DO_RETCODE(GNI_RC_NOT_DONE);
128                 DO_RETCODE(GNI_RC_INVALID_PARAM);
129                 DO_RETCODE(GNI_RC_ERROR_RESOURCE);
130                 DO_RETCODE(GNI_RC_TIMEOUT);
131                 DO_RETCODE(GNI_RC_PERMISSION_ERROR);
132                 DO_RETCODE(GNI_RC_DESCRIPTOR_ERROR);
133                 DO_RETCODE(GNI_RC_ALIGNMENT_ERROR);
134                 DO_RETCODE(GNI_RC_INVALID_STATE);
135                 DO_RETCODE(GNI_RC_NO_MATCH);
136                 DO_RETCODE(GNI_RC_SIZE_ERROR);
137                 DO_RETCODE(GNI_RC_TRANSACTION_ERROR);
138                 DO_RETCODE(GNI_RC_ILLEGAL_OP);
139                 DO_RETCODE(GNI_RC_ERROR_NOMEM);
140         }
141         LBUG();
142 }
143 #undef DO_RETCODE
144
145 /* log an error and LBUG for unhandled rc from gni api function
146  * the fmt should be something like:
147  *  gni_api_call(arg1, arg2, arg3)
148  */
149
150 /* apick_fn and apick_fmt should be defined for each site */
151 #undef apick_fn
152 #undef apick_fmt
153
154 #define GNILND_API_RC_LBUG(args...)                                             \
155 do {                                                                            \
156         LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);                     \
157         kgnilnd_api_rc_lbug(&msgdata, rrc, apick_fn"("apick_fmt")", ##args);    \
158 } while (0)
159
160 #define GNILND_API_SWBUG(args...)                                               \
161 do {                                                                            \
162         CERROR("likely SOFTWARE BUG "apick_fn"("apick_fmt") rc %s\n",           \
163                  ##args, kgnilnd_api_rc2str(rrc));                              \
164 } while (0)
165
166 #define GNILND_API_EINVAL(args...)                                              \
167 do {                                                                            \
168         CERROR("invalid parameter to "apick_fn"("apick_fmt") rc %s\n",          \
169                  ##args, kgnilnd_api_rc2str(rrc));                              \
170 } while (0)
171
172 #define GNILND_API_RESOURCE(args...)                                            \
173 do {                                                                            \
174         CERROR("no resources for "apick_fn"("apick_fmt") rc %s\n",              \
175                 ##args, kgnilnd_api_rc2str(rrc));                               \
176 } while (0)
177
178 #define GNILND_API_BUSY(args...)                                                \
179 do {                                                                            \
180         CERROR("resources busy for "apick_fn"("apick_fmt") rc %s\n",            \
181                 ##args, kgnilnd_api_rc2str(rrc));                               \
182 } while (0)
183
184 #undef DEBUG_SMSG_CREDITS
185 #ifdef DEBUG_SMSG_CREDITS
186 #define CRAY_CONFIG_GHAL_GEMINI
187 #include <gni_priv.h>
188 #define GNIDBG_SMSG_CREDS(level, conn)                                        \
189 do {                                                                          \
190         gni_ep_smsg_mbox_t *smsg = conn->gnc_ephandle->smsg;                  \
191         CDEBUG(level, "SMSGDBG: conn %p mcred %d/%d bcred %d/%d "             \
192                 "s_seq %d/%d/%d r_seq %d/%d/%d retr %d\n",                    \
193                 conn, smsg->mbox_credits, smsg->back_mbox_credits,            \
194                 smsg->buffer_credits, smsg->back_buffer_credits,              \
195                 smsg->s_seqno, smsg->s_seqno_back_mbox_credits,               \
196                 smsg->s_seqno_back_buffer_credits, smsg->r_seqno,             \
197                 smsg->r_seqno_back_mbox_credits,                              \
198                 smsg->r_seqno_back_buffer_credits, smsg->retransmit_count);   \
199 } while (0)
200 #else
201 #define GNIDBG_SMSG_CREDS(level, conn) do {} while(0)
202 #endif
203
204 /* these are all wrappers around gni_XXX functions.
205  * This allows us to handle all the return codes and api checks without
206  * dirtying up the logic code */
207
208 /* TODO: RETURN wrapper that translates integer to GNI API RC string */
209
210 #define apick_fn "kgnilnd_cdm_create"
211 #define apick_fmt "%u, %u, %u, %u, 0x%p"
212 static inline gni_return_t kgnilnd_cdm_create(
213                 IN uint32_t             inst_id,
214                 IN uint8_t              ptag,
215                 IN uint32_t             cookie,
216                 IN uint32_t             modes,
217                 OUT gni_cdm_handle_t    *cdm_hndl
218                 )
219 {
220         gni_return_t rrc;
221
222         /* Error injection */
223         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CDM_CREATE)) {
224                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
225         } else {
226                 rrc = gni_cdm_create(inst_id, ptag, cookie, modes, cdm_hndl);
227         }
228
229         switch (rrc)  {
230         case GNI_RC_SUCCESS:
231                 break;
232         case GNI_RC_ERROR_RESOURCE:
233         case GNI_RC_INVALID_PARAM:
234                 /* Try to bail gracefully */
235                 GNILND_API_SWBUG(
236                         inst_id, ptag, cookie, modes, cdm_hndl);
237                 break;
238         default:
239                 GNILND_API_RC_LBUG(
240                         inst_id, ptag, cookie, modes, cdm_hndl);
241
242                 /* LBUG never returns, but just for style and consistency */
243                 break;
244         }
245         RETURN(rrc);
246 }
247
248 #undef apick_fn
249 #undef apick_fmt
250
251 #define apick_fn "kgnilnd_cdm_attach"
252 #define apick_fmt "0x%p, %u, 0x%p, 0x%p"
253 static inline gni_return_t kgnilnd_cdm_attach(
254                 IN gni_cdm_handle_t     cdm_hndl,
255                 IN uint32_t             device_id,
256                 OUT uint32_t            *local_addr,
257                 OUT gni_nic_handle_t    *nic_hndl
258                 )
259 {
260         gni_return_t rrc;
261
262         /* Error injection */
263         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CDM_ATTACH)) {
264                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
265         } else {
266                 rrc = gni_cdm_attach(cdm_hndl, device_id, local_addr, nic_hndl);
267         }
268
269         switch (rrc)  {
270         case GNI_RC_SUCCESS:
271                 break;
272         case GNI_RC_NO_MATCH:
273         case GNI_RC_INVALID_PARAM:
274                 GNILND_API_SWBUG(
275                         cdm_hndl, device_id, local_addr, nic_hndl);
276                 break;
277         case GNI_RC_ERROR_RESOURCE:
278         case GNI_RC_INVALID_STATE:
279                 GNILND_API_RESOURCE(
280                         cdm_hndl, device_id, local_addr, nic_hndl);
281                 break;
282         default:
283                 GNILND_API_RC_LBUG(
284                         cdm_hndl, device_id, local_addr, nic_hndl);
285
286                 /* LBUG never returns, but just for style and consistency */
287                 break;
288         }
289         RETURN(rrc);
290 }
291 #undef apick_fmt
292 #undef apick_fn
293
294 #define apick_fn "kgnilnd_cdm_destroy"
295 #define apick_fmt "0x%p"
296 static inline gni_return_t kgnilnd_cdm_destroy(
297                 IN gni_cdm_handle_t     cdm_hndl
298                 )
299 {
300         gni_return_t rrc;
301
302         /* Error injection */
303         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_DESTROY)) {
304                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
305         } else {
306                 rrc = gni_cdm_destroy(
307                         cdm_hndl);
308         }
309
310         switch (rrc)  {
311         case GNI_RC_SUCCESS:
312                 break;
313         case GNI_RC_INVALID_PARAM:
314                 GNILND_API_SWBUG(
315                         cdm_hndl);
316                 break;
317         default:
318                 GNILND_API_RC_LBUG(
319                         cdm_hndl);
320
321                 /* LBUG never returns, but just for style and consistency */
322                 break;
323         }
324         RETURN(rrc);
325 }
326 #undef apick_fn
327 #undef apick_fmt
328
329 #define apick_fn "kgnilnd_subscribe_errors"
330 #define apick_fmt "0x%p,%x,%u,0x%p,0x%p,0x%p"
331 static inline gni_return_t kgnilnd_subscribe_errors(
332                 IN gni_nic_handle_t  nic_handle,
333                 IN gni_error_mask_t  mask,
334                 IN uint32_t          EEQ_size,
335                 IN void              (*EQ_new_event)(gni_err_handle_t),
336                 IN void              (*app_crit_err)(gni_err_handle_t),
337                 OUT gni_err_handle_t *err_handle
338                 )
339 {
340         gni_return_t rrc;
341
342         /* Error injection */
343         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_ERR_SUBSCRIBE)) {
344                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
345         } else {
346                 rrc = gni_subscribe_errors(
347                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
348                         err_handle);
349         }
350
351         switch (rrc)  {
352         case GNI_RC_SUCCESS:
353                 break;
354         case GNI_RC_INVALID_PARAM:
355                 GNILND_API_SWBUG(
356                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
357                         err_handle);
358                 break;
359         case GNI_RC_ERROR_RESOURCE:
360                 GNILND_API_RESOURCE(
361                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
362                         err_handle);
363                 break;
364         default:
365                 GNILND_API_RC_LBUG(
366                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
367                         err_handle);
368                 /* LBUG never returns, but just for style and consistency */
369                 break;
370         }
371         RETURN(rrc);
372 }
373 #undef apick_fn
374 #undef apick_fmt
375
376 #define apick_fn "kgnilnd_release_errors"
377 #define apick_fmt "0x%p"
378 static inline gni_return_t kgnilnd_release_errors(
379                 IN gni_err_handle_t err_handle
380                 )
381 {
382         gni_return_t rrc;
383
384         rrc = gni_release_errors(
385                         err_handle);
386
387         switch (rrc)  {
388         case GNI_RC_SUCCESS:
389                 break;
390         case GNI_RC_INVALID_PARAM:
391         case GNI_RC_NOT_DONE:
392                 GNILND_API_SWBUG(
393                         err_handle);
394                 break;
395         default:
396                 GNILND_API_RC_LBUG(
397                         err_handle);
398                 /* LBUG never returns, but just for style and consistency */
399                 break;
400         }
401         RETURN(rrc);
402 }
403 #undef apick_fn
404 #undef apick_fmt
405
406 #define apick_fn "kgnilnd_set_quiesce_callback"
407 #define apick_fmt "0x%p,0x%p"
408 static inline gni_return_t kgnilnd_set_quiesce_callback(
409                 IN gni_nic_handle_t  nic_handle,
410                 IN void              (*qsce_func)(gni_nic_handle_t, uint64_t msecs)
411                 )
412 {
413         gni_return_t rrc;
414
415         /* Error injection */
416         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_REG_QUIESCE)) {
417                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
418         } else {
419                 rrc = gni_set_quiesce_callback(
420                         nic_handle, qsce_func);
421         }
422
423         switch (rrc)  {
424         case GNI_RC_SUCCESS:
425                 break;
426         case GNI_RC_INVALID_STATE:
427         case GNI_RC_INVALID_PARAM:
428                 GNILND_API_SWBUG(
429                         nic_handle, qsce_func);
430                 break;
431         default:
432                 GNILND_API_RC_LBUG(
433                         nic_handle, qsce_func);
434                 /* LBUG never returns, but just for style and consistency */
435                 break;
436         }
437         RETURN(rrc);
438 }
439 #undef apick_fn
440 #undef apick_fmt
441
442 #define apick_fn "kgnilnd_get_quiesce_status"
443 #define apick_fmt "0x%p"
444 static inline gni_return_t kgnilnd_get_quiesce_status(
445                 IN gni_nic_handle_t  nic_handle
446                 )
447 {
448         uint32_t rrc;
449
450         /* this has weird RC -
451          * 0 - quiesce not in progress
452          * 1 - quiesce is turned on
453         */
454
455         /* Error injection */
456         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_IN_QUIESCE)) {
457                 rrc = 1;
458         } else {
459                 rrc = gni_get_quiesce_status(
460                         nic_handle);
461         }
462
463         switch (rrc)  {
464         case 1:
465         case 0:
466                 break;
467         default:
468                 GNILND_API_RC_LBUG(
469                         nic_handle);
470                 /* LBUG never returns, but just for style and consistency */
471                 break;
472         }
473         RETURN(rrc);
474 }
475 #undef apick_fn
476 #undef apick_fmt
477
478 #define apick_fn "kgnilnd_cq_create"
479 #define apick_fmt "0x%p, %u, %u, 0x%p, "LPX64", 0x%p"
480 static inline gni_return_t kgnilnd_cq_create(
481                 IN gni_nic_handle_t     nic_hndl,
482                 IN uint32_t             entry_count,
483                 IN uint32_t             delay_index,
484                 IN gni_cq_event_hndlr_f *event_handler,
485                 IN uint64_t             usr_event_data,
486                 OUT gni_cq_handle_t     *cq_hndl
487                 )
488 {
489         gni_return_t rrc;
490
491         /* Error injection */
492         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_CREATE)) {
493                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
494         } else {
495                 rrc = gni_cq_create(
496                        nic_hndl, entry_count, delay_index, event_handler,
497                         usr_event_data, cq_hndl);
498         }
499
500         switch (rrc)  {
501         case GNI_RC_SUCCESS:
502                 break;
503         case GNI_RC_INVALID_PARAM:
504                 GNILND_API_SWBUG(
505                         nic_hndl, entry_count, delay_index, event_handler,
506                         usr_event_data, cq_hndl);
507                 break;
508         case GNI_RC_ERROR_RESOURCE:
509                 GNILND_API_RESOURCE(
510                         nic_hndl, entry_count, delay_index, event_handler,
511                         usr_event_data, cq_hndl);
512                 break;
513         default:
514                 GNILND_API_RC_LBUG(
515                         nic_hndl, entry_count, delay_index, event_handler,
516                         usr_event_data, cq_hndl);
517
518                 /* LBUG never returns, but just for style and consistency */
519                 break;
520         }
521         RETURN(rrc);
522 }
523 #undef apick_fn
524 #undef apick_fmt
525
526 #define apick_fn "kgnilnd_cq_destroy"
527 #define apick_fmt "0x%p"
528 static inline gni_return_t kgnilnd_cq_destroy(
529                 IN gni_cq_handle_t cq_hndl
530                 )
531 {
532         gni_return_t rrc;
533
534         /* Error injection */
535         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_DESTROY)) {
536                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
537         } else {
538
539                 rrc = gni_cq_destroy(
540                         cq_hndl);
541         }
542
543         switch (rrc)  {
544         case GNI_RC_SUCCESS:
545                 break;
546         case GNI_RC_INVALID_PARAM:
547                 GNILND_API_SWBUG(
548                         cq_hndl);
549                 break;
550         case GNI_RC_ERROR_RESOURCE:
551                 GNILND_API_BUSY(
552                         cq_hndl);
553                 break;
554         default:
555                 GNILND_API_RC_LBUG(
556                         cq_hndl);
557
558                 /* LBUG never returns, but just for style and consistency */
559                 break;
560         }
561         RETURN(rrc);
562 }
563 #undef apick_fn
564 #undef apick_fmt
565
566 #define apick_fn "kgnilnd_cq_get_event"
567 #define apick_fmt "0x%p, 0x%p"
568 static inline gni_return_t kgnilnd_cq_get_event(
569                 IN gni_cq_handle_t cq_hndl,
570                 OUT gni_cq_entry_t *event_data
571                 )
572 {
573         gni_return_t rrc;
574
575         /* no error injection - CQs are touchy about the data.
576          * where appropriate, we'll do this on the CQs that should be able to
577          * handle the various errors */
578         rrc = gni_cq_get_event(
579                         cq_hndl, event_data);
580
581         switch (rrc)  {
582         case GNI_RC_SUCCESS:
583         case GNI_RC_NOT_DONE:
584         case GNI_RC_TRANSACTION_ERROR:
585                 break;
586         case GNI_RC_ERROR_RESOURCE:
587                 LASSERTF(GNI_CQ_OVERRUN(*event_data),
588                          "kgni returned ERROR_RESOURCE but cq_hndl 0x%p is not "
589                          "overrun\n", cq_hndl);
590                 break;
591         case GNI_RC_INVALID_PARAM:
592                 GNILND_API_SWBUG(
593                         cq_hndl, event_data);
594                 break;
595         default:
596                 GNILND_API_RC_LBUG(
597                         cq_hndl, event_data);
598
599                 /* LBUG never returns, but just for style and consistency */
600                 break;
601         }
602         return rrc;
603 }
604 #undef apick_fn
605 #undef apick_fmt
606
607 #define apick_fn "kgnilnd_smsg_init"
608 #define apick_fmt "0x%p, 0x%p, 0x%p"
609 static inline gni_return_t kgnilnd_smsg_init(
610                 IN gni_ep_handle_t      ep_hndl,
611                 IN gni_smsg_attr_t      *local_smsg_attr,
612                 IN gni_smsg_attr_t      *remote_smsg_attr
613                 )
614 {
615         gni_return_t rrc;
616
617         /* Error injection */
618         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_INIT)) {
619                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_RESOURCE;
620         } else {
621                 rrc = gni_smsg_init(
622                         ep_hndl, local_smsg_attr, remote_smsg_attr);
623         }
624
625         switch (rrc)  {
626         /* both of these are OK, upper SW needs to handle */
627         case GNI_RC_SUCCESS:
628         case GNI_RC_NOT_DONE:
629                 break;
630         case GNI_RC_INVALID_PARAM:
631         case GNI_RC_INVALID_STATE:
632                 GNILND_API_SWBUG(
633                         ep_hndl, local_smsg_attr, remote_smsg_attr);
634                 break;
635         case GNI_RC_ERROR_RESOURCE:
636                 GNILND_API_RESOURCE(
637                         ep_hndl, local_smsg_attr, remote_smsg_attr);
638                 break;
639         default:
640                 GNILND_API_RC_LBUG(
641                         ep_hndl, local_smsg_attr, remote_smsg_attr);
642
643                 /* LBUG never returns, but just for style and consistency */
644                 break;
645         }
646         RETURN(rrc);
647 }
648 #undef apick_fn
649 #undef apick_fmt
650
651 #define apick_fn "kgnilnd_smsg_send"
652 #define apick_fmt "0x%p, 0x%p, %d, 0x%p, %u %u"
653 static inline gni_return_t kgnilnd_smsg_send(
654                 IN gni_ep_handle_t      ep_hndl,
655                 IN void                 *header,
656                 IN uint32_t             header_length,
657                 IN void                 *data,
658                 IN uint32_t             data_length,
659                 IN uint32_t             msg_id
660                 )
661 {
662         gni_return_t rrc;
663
664         /* Error injection */
665         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_SEND)) {
666                 if (cfs_fail_loc & CFS_FAIL_RAND) {
667                         rrc = GNI_RC_NOT_DONE;
668                 } else {
669                         rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
670                 }
671         } else {
672                 rrc = gni_smsg_send(
673                         ep_hndl, header, header_length, data, data_length, msg_id);
674         }
675
676         switch (rrc)  {
677         /* both of these are OK, upper SW needs to handle */
678         case GNI_RC_SUCCESS:
679         case GNI_RC_NOT_DONE:
680                 break;
681         case GNI_RC_INVALID_PARAM:
682                 GNILND_API_SWBUG(
683                         ep_hndl, header, header_length, data, data_length, msg_id);
684                 break;
685         case GNI_RC_ERROR_RESOURCE:
686                 GNILND_API_RESOURCE(
687                         ep_hndl, header, header_length, data, data_length, msg_id);
688                 break;
689         default:
690                 GNILND_API_RC_LBUG(
691                         ep_hndl, header, header_length, data, data_length, msg_id);
692
693                 /* LBUG never returns, but just for style and consistency */
694                 break;
695         }
696         RETURN(rrc);
697 }
698 #undef apick_fn
699 #undef apick_fmt
700
701 #define apick_fn "kgnilnd_smsg_getnext"
702 #define apick_fmt "0x%p,0x%p"
703 static inline gni_return_t kgnilnd_smsg_getnext(
704                 IN gni_ep_handle_t      ep_hndl,
705                 OUT void                **header
706                 )
707 {
708         gni_return_t rrc;
709
710         /* Error injection */
711         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_RELEASE)) {
712                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
713         } else {
714                 rrc = gni_smsg_getnext(
715                         ep_hndl, header);
716         }
717
718         switch (rrc)  {
719         /* both of these are OK, upper SW needs to handle */
720         case GNI_RC_SUCCESS:
721         case GNI_RC_NOT_DONE:
722         case GNI_RC_INVALID_STATE:
723                 break;
724         case GNI_RC_INVALID_PARAM:
725                 GNILND_API_SWBUG(
726                         ep_hndl, header);
727                 break;
728         default:
729                 GNILND_API_RC_LBUG(
730                         ep_hndl, header);
731
732                 /* LBUG never returns, but just for style and consistency */
733                 break;
734         }
735         RETURN(rrc);
736 }
737 #undef apick_fn
738 #undef apick_fmt
739
740 #define apick_fn "kgnilnd_smsg_release"
741 #define apick_fmt "0x%p"
742 static inline gni_return_t kgnilnd_smsg_release(
743                 IN gni_ep_handle_t      ep_hndl
744                 )
745 {
746         gni_return_t rrc;
747
748         /* Error injection */
749         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_RELEASE)) {
750                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
751         } else {
752                 rrc = gni_smsg_release(
753                         ep_hndl);
754         }
755
756         switch (rrc)  {
757         /* both of these are OK, upper SW needs to handle */
758         case GNI_RC_SUCCESS:
759         case GNI_RC_NOT_DONE:
760                 break;
761         case GNI_RC_INVALID_PARAM:
762                 GNILND_API_SWBUG(
763                         ep_hndl);
764                 break;
765         default:
766                 GNILND_API_RC_LBUG(
767                         ep_hndl);
768
769                 /* LBUG never returns, but just for style and consistency */
770                 break;
771         }
772         RETURN(rrc);
773 }
774 #undef apick_fn
775 #undef apick_fmt
776
777 #define apick_fn "kgnilnd_ep_create"
778 #define apick_fmt "0x%p, 0x%p, 0x%p"
779 static inline gni_return_t kgnilnd_ep_create(
780                 IN gni_nic_handle_t     nic_hndl,
781                 IN gni_cq_handle_t      src_cq_hndl,
782                 OUT gni_ep_handle_t     *ep_hndl
783                 )
784 {
785         gni_return_t rrc;
786
787         /* error injection */
788         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_CREATE)) {
789                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_NOMEM;
790         } else {
791                 rrc = gni_ep_create(
792                         nic_hndl, src_cq_hndl, ep_hndl);
793         }
794
795         switch (rrc)  {
796         case GNI_RC_SUCCESS:
797                 break;
798         case GNI_RC_INVALID_PARAM:
799                 GNILND_API_SWBUG(
800                         nic_hndl, src_cq_hndl, ep_hndl);
801                 break;
802         case GNI_RC_ERROR_NOMEM:
803                 GNILND_API_RESOURCE(
804                         nic_hndl, src_cq_hndl, ep_hndl);
805                 break;
806         default:
807                 GNILND_API_RC_LBUG(
808                         nic_hndl, src_cq_hndl, ep_hndl);
809
810                 /* lbug never returns, but just for style and consistency */
811                 break;
812         }
813         RETURN(rrc);
814 }
815 #undef apick_fn
816 #undef apick_fmt
817
818 #define apick_fn "kgnilnd_ep_bind"
819 #define apick_fmt "0x%p, %x, %x"
820 static inline gni_return_t kgnilnd_ep_bind(
821                 IN gni_ep_handle_t      ep_hndl,
822                 IN uint32_t             remote_addr,
823                 IN uint32_t             remote_id
824                 )
825 {
826         gni_return_t rrc;
827
828         /* error injection */
829         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_BIND)) {
830                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
831         } else {
832                 rrc = gni_ep_bind(
833                         ep_hndl, remote_addr, remote_id);
834         }
835
836         switch (rrc)  {
837         /* both of these are ok, upper sw needs to handle */
838         case GNI_RC_SUCCESS:
839         case GNI_RC_NOT_DONE:
840                 break;
841         case GNI_RC_INVALID_PARAM:
842                 GNILND_API_SWBUG(
843                         ep_hndl, remote_addr, remote_id);
844                 break;
845         default:
846                 GNILND_API_RC_LBUG(
847                         ep_hndl, remote_addr, remote_id);
848
849                 /* lbug never returns, but just for style and consistency */
850                 break;
851         }
852         RETURN(rrc);
853 }
854 #undef apick_fn
855 #undef apick_fmt
856
857 #define apick_fn "kgnilnd_ep_set_eventdata"
858 #define apick_fmt "0x%p, %x, %x"
859 static inline gni_return_t kgnilnd_ep_set_eventdata(
860                 IN gni_ep_handle_t      ep_hndl,
861                 IN uint32_t             local_event,
862                 IN uint32_t             remote_event
863                 )
864 {
865         gni_return_t rrc;
866
867         /* Error injection */
868         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_SET_EVDATA)) {
869                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
870         } else {
871                 rrc = gni_ep_set_eventdata(
872                         ep_hndl, local_event, remote_event);
873         }
874
875         switch (rrc)  {
876         case GNI_RC_SUCCESS:
877                 break;
878         case GNI_RC_INVALID_PARAM:
879                 GNILND_API_SWBUG(
880                         ep_hndl, local_event, remote_event);
881                 break;
882         default:
883                 GNILND_API_RC_LBUG(
884                         ep_hndl, local_event, remote_event);
885
886                 /* LBUG never returns, but just for style and consistency */
887                 break;
888         }
889         RETURN(rrc);
890 }
891 #undef apick_fn
892 #undef apick_fmt
893
894 #define apick_fn "kgnilnd_ep_unbind"
895 #define apick_fmt "0x%p"
896 static inline gni_return_t kgnilnd_ep_unbind(
897                 IN gni_ep_handle_t      ep_hndl
898                 )
899 {
900         gni_return_t rrc;
901
902         /* Error injection */
903         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_UNBIND)) {
904                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
905         } else {
906                 rrc = gni_ep_unbind(
907                         ep_hndl);
908         }
909
910         switch (rrc)  {
911         /* both of these are OK, upper SW needs to handle */
912         case GNI_RC_NOT_DONE:
913         case GNI_RC_SUCCESS:
914                 break;
915         case GNI_RC_INVALID_PARAM:
916                 GNILND_API_SWBUG(
917                         ep_hndl);
918                 break;
919         default:
920                 GNILND_API_RC_LBUG(
921                         ep_hndl);
922
923                 /* LBUG never returns, but just for style and consistency */
924                 break;
925         }
926         RETURN(rrc);
927 }
928 #undef apick_fn
929 #undef apick_fmt
930
931 #define apick_fn "kgnilnd_ep_destroy"
932 #define apick_fmt "0x%p"
933 static inline gni_return_t kgnilnd_ep_destroy(
934                 IN gni_ep_handle_t      ep_hndl
935                 )
936 {
937         gni_return_t rrc;
938
939         /* Error injection */
940         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_DESTROY)) {
941                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
942         } else {
943                 rrc = gni_ep_destroy(
944                         ep_hndl);
945         }
946
947         switch (rrc)  {
948         case GNI_RC_SUCCESS:
949                 break;
950         case GNI_RC_INVALID_PARAM:
951                 GNILND_API_SWBUG(
952                         ep_hndl);
953                 break;
954         default:
955                 GNILND_API_RC_LBUG(
956                         ep_hndl);
957
958                 /* LBUG never returns, but just for style and consistency */
959                 break;
960         }
961         RETURN(rrc);
962 }
963 #undef apick_fn
964 #undef apick_fmt
965
966 #define apick_fn "kgnilnd_ep_postdata_w_id"
967 #define apick_fmt "0x%p, 0x%p, %d, 0x%p, %d, "LPU64""
968 static inline gni_return_t kgnilnd_ep_postdata_w_id(
969                 IN gni_ep_handle_t ep_hndl,
970                 IN void            *in_data,
971                 IN uint16_t        data_len,
972                 IN void            *out_buf,
973                 IN uint16_t        buf_size,
974                 IN uint64_t        datagram_id
975                 )
976 {
977         gni_return_t rrc;
978
979         /* Error injection */
980         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_POST)) {
981                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_SIZE_ERROR;
982         } else {
983                 rrc = gni_ep_postdata_w_id(
984                         ep_hndl, in_data, data_len, out_buf, buf_size,
985                         datagram_id);
986         }
987
988         switch (rrc)  {
989         case GNI_RC_SUCCESS:
990         case GNI_RC_ERROR_NOMEM:
991         case GNI_RC_ERROR_RESOURCE:
992                 break;
993         case GNI_RC_INVALID_PARAM:
994         case GNI_RC_SIZE_ERROR:
995                 GNILND_API_SWBUG(
996                         ep_hndl, in_data, data_len, out_buf, buf_size,
997                         datagram_id);
998                 break;
999         default:
1000                 GNILND_API_RC_LBUG(
1001                         ep_hndl, in_data, data_len, out_buf, buf_size,
1002                         datagram_id);
1003
1004                 /* LBUG never returns, but just for style and consistency */
1005                 break;
1006         }
1007         RETURN(rrc);
1008 }
1009 #undef apick_fn
1010 #undef apick_fmt
1011
1012 #define apick_fn "kgnilnd_ep_postdata_test_by_id"
1013 #define apick_fmt "0x%p, "LPU64", 0x%p, 0x%p, 0x%p"
1014 static inline gni_return_t kgnilnd_ep_postdata_test_by_id(
1015                 IN gni_ep_handle_t      ep_hndl,
1016                 IN uint64_t             datagram_id,
1017                 OUT gni_post_state_t    *post_state,
1018                 OUT uint32_t            *remote_addr,
1019                 OUT uint32_t            *remote_id
1020                 )
1021 {
1022         gni_return_t rrc;
1023
1024         /* Error injection */
1025         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_TEST)) {
1026                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_NOMEM;
1027         } else {
1028                 rrc = gni_ep_postdata_test_by_id(
1029                         ep_hndl, datagram_id, post_state, remote_addr,
1030                         remote_id);
1031
1032                 /* we want to lie, but we need to do the actual work first
1033                  * so we don't keep getting the event saying a dgram is ready */
1034                 if (rrc == GNI_RC_SUCCESS && CFS_FAIL_CHECK(CFS_FAIL_GNI_DG_TERMINATE)) {
1035                         /* don't use fail_val, allows us to do FAIL_SOME */
1036                         *post_state = GNI_POST_TERMINATED;
1037                 }
1038         }
1039
1040         switch (rrc)  {
1041         case GNI_RC_SUCCESS:
1042         case GNI_RC_NO_MATCH:
1043                 break;
1044         case GNI_RC_SIZE_ERROR:
1045         case GNI_RC_INVALID_PARAM:
1046                 GNILND_API_SWBUG(
1047                         ep_hndl, datagram_id, post_state, remote_addr,
1048                         remote_id);
1049                 break;
1050         case GNI_RC_ERROR_NOMEM:
1051                 GNILND_API_RESOURCE(
1052                         ep_hndl, datagram_id, post_state, remote_addr,
1053                         remote_id);
1054                 break;
1055         default:
1056                 GNILND_API_RC_LBUG(
1057                         ep_hndl, datagram_id, post_state, remote_addr,
1058                         remote_id);
1059
1060                 /* LBUG never returns, but just for style and consistency */
1061                 break;
1062         }
1063         RETURN(rrc);
1064 }
1065 #undef apick_fn
1066 #undef apick_fmt
1067
1068 #define apick_fn "kgnilnd_ep_postdata_cancel_by_id"
1069 #define apick_fmt "0x%p, "LPU64""
1070 static inline gni_return_t kgnilnd_ep_postdata_cancel_by_id(
1071                 IN gni_ep_handle_t      ep_hndl,
1072                 IN uint64_t             datagram_id
1073                 )
1074 {
1075         gni_return_t rrc;
1076
1077         /* no error injection as the only thing we'd do is LBUG */
1078
1079         rrc = gni_ep_postdata_cancel_by_id(
1080                 ep_hndl, datagram_id);
1081
1082         switch (rrc)  {
1083         case GNI_RC_SUCCESS:
1084         case GNI_RC_NO_MATCH:
1085                 break;
1086         case GNI_RC_INVALID_PARAM:
1087                 GNILND_API_SWBUG(
1088                         ep_hndl, datagram_id);
1089                 break;
1090         default:
1091                 GNILND_API_RC_LBUG(
1092                         ep_hndl, datagram_id);
1093
1094                 /* LBUG never returns, but just for style and consistency */
1095                 break;
1096         }
1097         RETURN(rrc);
1098 }
1099 #undef apick_fn
1100 #undef apick_fmt
1101
1102 #define apick_fn "kgnilnd_postdata_probe_by_id"
1103 #define apick_fmt "0x%p, 0x%p"
1104 static inline gni_return_t kgnilnd_postdata_probe_by_id(
1105                 IN gni_nic_handle_t    nic_hndl,
1106                 OUT uint64_t          *datagram_id
1107                 )
1108 {
1109         gni_return_t rrc;
1110
1111         /* Error injection */
1112         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PROBE)) {
1113                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NO_MATCH;
1114         } else {
1115                 rrc = gni_postdata_probe_by_id(
1116                         nic_hndl, datagram_id);
1117         }
1118
1119         switch (rrc)  {
1120         case GNI_RC_SUCCESS:
1121         case GNI_RC_NO_MATCH:
1122                 break;
1123         case GNI_RC_INVALID_PARAM:
1124                 GNILND_API_SWBUG(
1125                         nic_hndl, datagram_id);
1126                 break;
1127         default:
1128                 GNILND_API_RC_LBUG(
1129                         nic_hndl, datagram_id);
1130
1131                 /* LBUG never returns, but just for style and consistency */
1132                 break;
1133         }
1134         RETURN(rrc);
1135 }
1136 #undef apick_fn
1137 #undef apick_fmt
1138
1139 #define apick_fn "kgnilnd_postdata_probe_wait_by_id"
1140 #define apick_fmt "0x%p, %d, 0x%p"
1141 static inline gni_return_t kgnilnd_postdata_probe_wait_by_id(
1142                 IN gni_nic_handle_t nic_hndl,
1143                 IN uint32_t         timeout,
1144                 OUT uint64_t        *datagram_id
1145                 )
1146 {
1147         gni_return_t rrc;
1148
1149         /* Error injection */
1150         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PROBE_WAIT)) {
1151                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_TIMEOUT;
1152         } else {
1153                 rrc = gni_postdata_probe_wait_by_id(
1154                         nic_hndl, timeout, datagram_id);
1155         }
1156
1157         switch (rrc)  {
1158         case GNI_RC_SUCCESS:
1159         case GNI_RC_TIMEOUT:
1160                 break;
1161         case GNI_RC_INVALID_PARAM:
1162                 GNILND_API_SWBUG(
1163                         nic_hndl, timeout, datagram_id);
1164                 break;
1165         default:
1166                 GNILND_API_RC_LBUG(
1167                         nic_hndl, timeout, datagram_id);
1168
1169                 /* LBUG never returns, but just for style and consistency */
1170                 break;
1171         }
1172         RETURN(rrc);
1173 }
1174 #undef apick_fn
1175 #undef apick_fmt
1176
1177 #define apick_fn "kgnilnd_post_rdma"
1178 #define apick_fmt "0x%p, 0x%p"
1179 static inline gni_return_t kgnilnd_post_rdma(
1180                 IN gni_ep_handle_t               ep_hndl,
1181                 IN gni_post_descriptor_t        *post_descr
1182                 )
1183 {
1184         gni_return_t rrc;
1185
1186         /* Error injection */
1187         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_POST_RDMA)) {
1188                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
1189         } else {
1190                 rrc = gni_post_rdma(
1191                         ep_hndl, post_descr);
1192         }
1193
1194         switch (rrc)  {
1195         case GNI_RC_SUCCESS:
1196                 break;
1197         case GNI_RC_ALIGNMENT_ERROR:
1198         case GNI_RC_INVALID_PARAM:
1199                 GNILND_API_SWBUG(
1200                         ep_hndl, post_descr);
1201                 break;
1202         case GNI_RC_ERROR_RESOURCE:
1203                 CDEBUG(D_NET, "no resources for kgnilnd_post_rdma (0x%p, 0x%p)"
1204                         " rc %s\n", ep_hndl, post_descr,
1205                         kgnilnd_api_rc2str(rrc));
1206                 break;
1207         default:
1208                 GNILND_API_RC_LBUG(
1209                         ep_hndl, post_descr);
1210
1211                 /* LBUG never returns, but just for style and consistency */
1212                 break;
1213         }
1214         RETURN(rrc);
1215 }
1216 #undef apick_fn
1217 #undef apick_fmt
1218
1219 #define apick_fn "kgnilnd_get_completed"
1220 #define apick_fmt "0x%p,"LPX64",0x%p"
1221 static inline gni_return_t kgnilnd_get_completed(
1222                 IN gni_cq_handle_t              cq_hndl,
1223                 IN gni_cq_entry_t               event_data,
1224                 OUT gni_post_descriptor_t       **post_descr
1225                 )
1226 {
1227         gni_return_t rrc;
1228
1229
1230         rrc = gni_get_completed(cq_hndl, event_data, post_descr);
1231
1232         switch (rrc)  {
1233         case GNI_RC_TRANSACTION_ERROR:
1234         case GNI_RC_SUCCESS:
1235                 break;
1236         case GNI_RC_DESCRIPTOR_ERROR:
1237         case GNI_RC_INVALID_PARAM:
1238                 GNILND_API_SWBUG(cq_hndl, event_data, post_descr);
1239                 break;
1240         default:
1241                 GNILND_API_RC_LBUG(cq_hndl, event_data, post_descr);
1242                 /* LBUG never returns, but just for style and consistency */
1243                 break;
1244         }
1245
1246         /* Error injection - we need a valid desc, so let kgni give us one
1247          * - then we lie  */
1248         if (rrc == GNI_RC_SUCCESS &&
1249             (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED))) {
1250                 /* We only trigger TRANSACTION_ERROR for now */
1251                 gni_post_descriptor_t *desc;
1252                 rrc = GNI_RC_TRANSACTION_ERROR;
1253                 desc = *post_descr;
1254                 desc->status = rrc;
1255                 /* recoverable decision made from cfs_fail_val in
1256                  *  kgnilnd_cq_error_str and
1257                  *  kgnilnd_cq_error_recoverable */
1258         }
1259         RETURN(rrc);
1260 }
1261 #undef apick_fn
1262 #undef apick_fmt
1263
1264 #define apick_fn "kgnilnd_cq_error_str"
1265 #define apick_fmt LPX64",0x%p,%d"
1266 static inline gni_return_t kgnilnd_cq_error_str(
1267                 IN gni_cq_entry_t       entry,
1268                 IN void                *buffer,
1269                 IN uint32_t             len
1270                 )
1271 {
1272         gni_return_t rrc;
1273
1274         /* Error injection - set string if we injected a
1275          *  TRANSACTION_ERROR earlier */
1276         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED)) {
1277                 /* if we just set persistent error, we can't ever
1278                  * break in via ssh to clear, so use a count > 10 to indicate fatal */
1279                 sprintf(buffer, "INJECT:%s", cfs_fail_val > 10 ?
1280                         "FATAL" : "RECOVERABLE");
1281                 rrc = GNI_RC_SUCCESS;
1282         } else {
1283                 rrc = gni_cq_error_str(
1284                         entry, buffer, len);
1285         }
1286
1287         switch (rrc)  {
1288         case GNI_RC_SUCCESS:
1289                 break;
1290         case GNI_RC_SIZE_ERROR:
1291         case GNI_RC_INVALID_PARAM:
1292                 GNILND_API_SWBUG(
1293                         entry, buffer, len);
1294                 /* give them something to use */
1295                 snprintf(buffer, len, "UNDEF:UNDEF");
1296                 break;
1297         default:
1298                 GNILND_API_RC_LBUG(
1299                         entry, buffer, len);
1300
1301                 /* LBUG never returns, but just for style and consistency */
1302                 break;
1303         }
1304         RETURN(rrc);
1305 }
1306 #undef apick_fn
1307 #undef apick_fmt
1308
1309 #define apick_fn "kgnilnd_cq_error_recoverable"
1310 #define apick_fmt LPX64",0x%p"
1311 static inline gni_return_t kgnilnd_cq_error_recoverable(
1312                 IN gni_cq_entry_t       entry,
1313                 IN uint32_t            *recoverable
1314                 )
1315 {
1316         gni_return_t rrc;
1317
1318         /* Error injection - set string if we injected a
1319          *  TRANSACTION_ERROR earlier */
1320         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED)) {
1321                 *recoverable = cfs_fail_val > 10 ? 0 : 1;
1322                 rrc = GNI_RC_SUCCESS;
1323         } else {
1324                 rrc = gni_cq_error_recoverable(
1325                         entry, recoverable);
1326         }
1327
1328         switch (rrc)  {
1329         case GNI_RC_SUCCESS:
1330                 break;
1331         case GNI_RC_INVALID_STATE:
1332         case GNI_RC_INVALID_PARAM:
1333                 GNILND_API_SWBUG(
1334                         entry, recoverable);
1335                 *recoverable = 0;
1336                 break;
1337         default:
1338                 GNILND_API_RC_LBUG(
1339                         entry, recoverable);
1340
1341                 /* LBUG never returns, but just for style and consistency */
1342                 break;
1343         }
1344         RETURN(rrc);
1345 }
1346 #undef apick_fn
1347 #undef apick_fmt
1348
1349 #define apick_fn "kgnilnd_mem_register_segments"
1350 #define apick_fmt "0x%p,0x%p,%u,0x%p,%x,0x%p"
1351 static inline gni_return_t
1352 kgnilnd_mem_register_segments(
1353                 IN gni_nic_handle_t     nic_hndl,
1354                 IN gni_mem_segment_t    *mem_segments,
1355                 IN uint32_t             segments_cnt,
1356                 IN gni_cq_handle_t      dst_cq_hndl,
1357                 IN uint32_t             flags,
1358                 OUT gni_mem_handle_t    *mem_hndl
1359                 )
1360 {
1361         gni_return_t rrc;
1362
1363         /* Error injection */
1364         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PHYS_MAP)) {
1365                 rrc = GNI_RC_ERROR_RESOURCE;
1366         } else {
1367                 rrc = gni_mem_register_segments(
1368                         nic_hndl, mem_segments, segments_cnt,
1369                         dst_cq_hndl, flags, mem_hndl);
1370         }
1371
1372         switch (rrc)  {
1373         case GNI_RC_SUCCESS:
1374         case GNI_RC_ERROR_RESOURCE:
1375                 break;
1376         case GNI_RC_INVALID_PARAM:
1377                 GNILND_API_SWBUG(
1378                         nic_hndl, mem_segments, segments_cnt,
1379                         dst_cq_hndl, flags, mem_hndl);
1380                 break;
1381         default:
1382                 GNILND_API_RC_LBUG(
1383                         nic_hndl, mem_segments, segments_cnt,
1384                         dst_cq_hndl, flags, mem_hndl);
1385
1386                 /* LBUG never returns, but just for style and consistency */
1387                 break;
1388         }
1389         RETURN(rrc);
1390 }
1391 #undef apick_fn
1392 #undef apick_fmt
1393
1394 #define apick_fn "kgnilnd_mem_register"
1395 #define apick_fmt "0x%p,"LPX64","LPX64"0x%p,%u,0x%p"
1396 static inline gni_return_t kgnilnd_mem_register(
1397                 IN gni_nic_handle_t     nic_hndl,
1398                 IN uint64_t             address,
1399                 IN uint64_t             length,
1400                 IN gni_cq_handle_t      dst_cq_hndl,
1401                 IN uint32_t             flags,
1402                 OUT gni_mem_handle_t    *mem_hndl
1403                 )
1404 {
1405         gni_return_t rrc;
1406
1407         /* Error injection */
1408         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_MAP)) {
1409                 rrc = GNI_RC_ERROR_RESOURCE;
1410         } else if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_SMALL_MAP) &&
1411                    length <= *kgnilnd_tunables.kgn_max_immediate) {
1412                 rrc = GNI_RC_INVALID_PARAM;
1413         } else {
1414                 rrc = gni_mem_register(
1415                         nic_hndl, address, length,
1416                         dst_cq_hndl, flags, mem_hndl);
1417         }
1418
1419         switch (rrc)  {
1420         case GNI_RC_SUCCESS:
1421         case GNI_RC_ERROR_RESOURCE:
1422                 break;
1423         case GNI_RC_INVALID_PARAM:
1424                 GNILND_API_SWBUG(
1425                         nic_hndl, address, length,
1426                         dst_cq_hndl, flags, mem_hndl);
1427                 break;
1428         default:
1429                 GNILND_API_RC_LBUG(
1430                         nic_hndl, address, length,
1431                         dst_cq_hndl, flags, mem_hndl);
1432
1433                 /* LBUG never returns, but just for style and consistency */
1434                 break;
1435         }
1436         RETURN(rrc);
1437 }
1438 #undef apick_fn
1439 #undef apick_fmt
1440
1441 #define apick_fn "kgnilnd_mem_deregister"
1442 #define apick_fmt "0x%p,0x%p,%d"
1443 static inline gni_return_t kgnilnd_mem_deregister(
1444                 IN gni_nic_handle_t     nic_hndl,
1445                 IN gni_mem_handle_t     *mem_hndl,
1446                 IN int                  hold_timeout
1447                 )
1448 {
1449         gni_return_t rrc;
1450
1451         /* Error injection */
1452         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_UNMAP)) {
1453                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
1454         } else {
1455                 rrc = gni_mem_deregister(
1456                         nic_hndl, mem_hndl, hold_timeout);
1457         }
1458
1459         switch (rrc)  {
1460         case GNI_RC_SUCCESS:
1461                 break;
1462         case GNI_RC_INVALID_PARAM:
1463                 GNILND_API_SWBUG(
1464                         nic_hndl, mem_hndl, hold_timeout);
1465                 break;
1466         default:
1467                 GNILND_API_RC_LBUG(
1468                         nic_hndl, mem_hndl, hold_timeout);
1469
1470                 /* LBUG never returns, but just for style and consistency */
1471                 break;
1472         }
1473         RETURN(rrc);
1474 }
1475 #undef apick_fn
1476 #undef apick_fmt
1477
1478 #define apick_fn "kgnilnd_mem_mdd_release"
1479 #define apick_fmt "0x%p,0x%p"
1480 static inline gni_return_t kgnilnd_mem_mdd_release(
1481                 IN gni_nic_handle_t     nic_hndl,
1482                 IN gni_mem_handle_t     *mem_hndl
1483                 )
1484 {
1485         gni_return_t rrc;
1486
1487         /* Error injection */
1488         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_MDD_RELEASE)) {
1489                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NO_MATCH;
1490         } else {
1491                 rrc = gni_mem_mdd_release(
1492                         nic_hndl, mem_hndl);
1493         }
1494
1495         switch (rrc)  {
1496         case GNI_RC_SUCCESS:
1497         case GNI_RC_NO_MATCH:
1498                 break;
1499         default:
1500                 GNILND_API_RC_LBUG(
1501                         nic_hndl, mem_hndl);
1502
1503                 /* LBUG never returns, but just for style and consistency */
1504                 break;
1505         }
1506         RETURN(rrc);
1507 }
1508 #undef apick_fn
1509 #undef apick_fmt
1510
1511 #endif /* _GNILND_API_WRAP_H */