lnet/klnds/gnilnd/gnilnd_api_wrap.h

   1 /*
   2  * Copyright (C) 2009-2012 Cray, Inc.
   3  *
   4  *   Author: Nic Henke <nic@cray.com>
   5  *
   6  *   This file is part of Lustre, http://www.lustre.org.
   7  *
   8  *   Lustre is free software; you can redistribute it and/or
   9  *   modify it under the terms of version 2 of the GNU General Public
  10  *   License as published by the Free Software Foundation.
  11  *
  12  *   Lustre is distributed in the hope that it will be useful,
  13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *   GNU General Public License for more details.
  16  *
  17  *   You should have received a copy of the GNU General Public License
  18  *   along with Lustre; if not, write to the Free Software
  19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20  *
  21  */
  22 #ifndef _GNILND_API_WRAP_H
  23 #define _GNILND_API_WRAP_H
  24
  25 /* LNet is allocated failure locations 0xe000 to 0xffff */
  26
  27 /* GNILND has 0xf0XX */
  28 #define CFS_FAIL_GNI                    0xf000
  29 #define CFS_FAIL_GNI_PHYS_MAP           0xf001
  30 #define CFS_FAIL_GNI_VIRT_MAP           0xf002
  31 #define CFS_FAIL_GNI_GET_UNMAP          0xf003
  32 #define CFS_FAIL_GNI_PUT_UNMAP          0xf004
  33 #define CFS_FAIL_GNI_MAP_TX             0xf005
  34 #define CFS_FAIL_GNI_SMSG_SEND          0xf006
  35 #define CFS_FAIL_GNI_CLOSE_SEND         0xf007
  36 #define CFS_FAIL_GNI_CDM_CREATE         0xf008
  37 #define CFS_FAIL_GNI_CDM_DESTROY        0xf009
  38 #define CFS_FAIL_GNI_CDM_ATTACH         0xf00a
  39 #define CFS_FAIL_GNI_CQ_CREATE          0xf00b
  40 #define CFS_FAIL_GNI_CQ_DESTROY         0xf00c
  41 #define CFS_FAIL_GNI_EP_BIND            0xf00d
  42 #define CFS_FAIL_GNI_EP_UNBIND          0xf00e
  43 #define CFS_FAIL_GNI_EP_SET_EVDATA      0xf00f
  44 #define CFS_FAIL_GNI_SMSG_INIT          0xf010
  45 #define CFS_FAIL_GNI_SMSG_RELEASE       0xf011
  46 #define CFS_FAIL_GNI_POST_RDMA          0xf012
  47 #define CFS_FAIL_GNI_GET_COMPLETED      0xf013
  48 #define CFS_FAIL_GNI_EP_DESTROY         0xf015
  49 #define CFS_FAIL_GNI_VIRT_UNMAP         0xf016
  50 #define CFS_FAIL_GNI_MDD_RELEASE        0xf017
  51 #define CFS_FAIL_GNI_NOOP_SEND          0xf018
  52 #define CFS_FAIL_GNI_ERR_SUBSCRIBE      0xf01a
  53 #define CFS_FAIL_GNI_QUIESCE_RACE       0xf01b
  54 #define CFS_FAIL_GNI_DG_TERMINATE       0xf01c
  55 #define CFS_FAIL_GNI_REG_QUIESCE        0xf01d
  56 #define CFS_FAIL_GNI_IN_QUIESCE         0xf01e
  57 #define CFS_FAIL_GNI_DELAY_RDMA         0xf01f
  58 #define CFS_FAIL_GNI_SR_DOWN_RACE       0xf020
  59 #define CFS_FAIL_GNI_ALLOC_TX           0xf021
  60 #define CFS_FAIL_GNI_FMABLK_AVAIL       0xf022
  61 #define CFS_FAIL_GNI_EP_CREATE          0xf023
  62 #define CFS_FAIL_GNI_CQ_GET_EVENT       0xf024
  63 #define CFS_FAIL_GNI_PROBE              0xf025
  64 #define CFS_FAIL_GNI_EP_TEST            0xf026
  65 #define CFS_FAIL_GNI_CONNREQ_DROP       0xf027
  66 #define CFS_FAIL_GNI_CONNREQ_PROTO      0xf028
  67 #define CFS_FAIL_GNI_CONND_PILEUP       0xf029
  68 #define CFS_FAIL_GNI_PHYS_SETUP         0xf02a
  69 #define CFS_FAIL_GNI_FIND_TARGET        0xf02b
  70 #define CFS_FAIL_GNI_WC_DGRAM_FREE      0xf02c
  71 #define CFS_FAIL_GNI_DROP_CLOSING       0xf02d
  72 #define CFS_FAIL_GNI_RX_CLOSE_CLOSING   0xf02e
  73 #define CFS_FAIL_GNI_RX_CLOSE_CLOSED    0xf02f
  74 #define CFS_FAIL_GNI_EP_POST            0xf030
  75 #define CFS_FAIL_GNI_PACK_SRCNID        0xf031
  76 #define CFS_FAIL_GNI_PACK_DSTNID        0xf032
  77 #define CFS_FAIL_GNI_PROBE_WAIT         0xf033
  78 #define CFS_FAIL_GNI_SMSG_CKSUM1        0xf034
  79 #define CFS_FAIL_GNI_SMSG_CKSUM2        0xf035
  80 #define CFS_FAIL_GNI_SMSG_CKSUM3        0xf036
  81 #define CFS_FAIL_GNI_DROP_DESTROY_EP    0xf037
  82 #define CFS_FAIL_GNI_SMSG_GETNEXT       0xf038
  83 #define CFS_FAIL_GNI_FINISH_PURG        0xf039
  84 #define CFS_FAIL_GNI_PURG_REL_DELAY     0xf03a
  85 #define CFS_FAIL_GNI_DONT_NOTIFY        0xf03b
  86 #define CFS_FAIL_GNI_VIRT_SMALL_MAP     0xf03c
  87 #define CFS_FAIL_GNI_DELAY_RDMAQ        0xf03d
  88 #define CFS_FAIL_GNI_PAUSE_SHUTDOWN     0xf03e
  89 #define CFS_FAIL_GNI_PAUSE_DGRAM_COMP   0xf03f
  90 #define CFS_FAIL_GNI_NET_LOOKUP         0xf040
  91 #define CFS_FAIL_GNI_RECV_TIMEOUT       0xf041
  92 #define CFS_FAIL_GNI_SEND_TIMEOUT       0xf042
  93 #define CFS_FAIL_GNI_ONLY_NOOP          0xf043
  94 #define CFS_FAIL_GNI_FINISH_PURG2       0xf044
  95 #define CFS_FAIL_GNI_RACE_RESET         0xf045
  96 #define CFS_FAIL_GNI_GNP_CONNECTING1    0xf046
  97 #define CFS_FAIL_GNI_GNP_CONNECTING2    0xf047
  98 #define CFS_FAIL_GNI_GNP_CONNECTING3    0xf048
  99 #define CFS_FAIL_GNI_SCHEDULE_COMPLETE  0xf049
 100 #define CFS_FAIL_GNI_PUT_ACK_AGAIN      0xf050
 101 #define CFS_FAIL_GNI_GET_REQ_AGAIN      0xf051
 102 #define CFS_FAIL_GNI_SCHED_DEADLINE     0xf052
 103 #define CFS_FAIL_GNI_DGRAM_DEADLINE     0xf053
 104
 105
 106 /* helper macros */
 107 extern void
 108 _kgnilnd_api_rc_lbug(const char *rcstr, int rc, struct libcfs_debug_msg_data *data,
 109                         const char *fmt, ...)
 110         __attribute__ ((format (printf, 4, 5)));
 111
 112 #define kgnilnd_api_rc_lbug(msgdata, rc, fmt, a...)                             \
 113 do {                                                                            \
 114         CFS_CHECK_STACK(msgdata, D_ERROR, NULL);                                \
 115         /* we don't mask this - it is always at D_ERROR */                      \
 116         _kgnilnd_api_rc_lbug(kgnilnd_api_rc2str(rc), (rc), msgdata, fmt, ##a);  \
 117 } while (0)
 118
 119 #define DO_RETCODE(x) case x: return #x;
 120 static inline const char *
 121 kgnilnd_api_rc2str(gni_return_t rrc)
 122 {
 123
 124         switch (rrc) {
 125                 DO_RETCODE(GNI_RC_SUCCESS)
 126                 DO_RETCODE(GNI_RC_NOT_DONE);
 127                 DO_RETCODE(GNI_RC_INVALID_PARAM);
 128                 DO_RETCODE(GNI_RC_ERROR_RESOURCE);
 129                 DO_RETCODE(GNI_RC_TIMEOUT);
 130                 DO_RETCODE(GNI_RC_PERMISSION_ERROR);
 131                 DO_RETCODE(GNI_RC_DESCRIPTOR_ERROR);
 132                 DO_RETCODE(GNI_RC_ALIGNMENT_ERROR);
 133                 DO_RETCODE(GNI_RC_INVALID_STATE);
 134                 DO_RETCODE(GNI_RC_NO_MATCH);
 135                 DO_RETCODE(GNI_RC_SIZE_ERROR);
 136                 DO_RETCODE(GNI_RC_TRANSACTION_ERROR);
 137                 DO_RETCODE(GNI_RC_ILLEGAL_OP);
 138                 DO_RETCODE(GNI_RC_ERROR_NOMEM);
 139         }
 140         LBUG();
 141 }
 142 #undef DO_RETCODE
 143
 144 /* log an error and LBUG for unhandled rc from gni api function
 145  * the fmt should be something like:
 146  *  gni_api_call(arg1, arg2, arg3)
 147  */
 148
 149 /* apick_fn and apick_fmt should be defined for each site */
 150 #undef apick_fn
 151 #undef apick_fmt
 152
 153 #define GNILND_API_RC_LBUG(args...)                                             \
 154 do {                                                                            \
 155         LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);                     \
 156         kgnilnd_api_rc_lbug(&msgdata, rrc, apick_fn"("apick_fmt")", ##args);    \
 157 } while (0)
 158
 159 #define GNILND_API_SWBUG(args...)                                               \
 160 do {                                                                            \
 161         CERROR("likely SOFTWARE BUG "apick_fn"("apick_fmt") rc %s\n",           \
 162                  ##args, kgnilnd_api_rc2str(rrc));                              \
 163 } while (0)
 164
 165 #define GNILND_API_EINVAL(args...)                                              \
 166 do {                                                                            \
 167         CERROR("invalid parameter to "apick_fn"("apick_fmt") rc %s\n",          \
 168                  ##args, kgnilnd_api_rc2str(rrc));                              \
 169 } while (0)
 170
 171 #define GNILND_API_RESOURCE(args...)                                            \
 172 do {                                                                            \
 173         CERROR("no resources for "apick_fn"("apick_fmt") rc %s\n",              \
 174                 ##args, kgnilnd_api_rc2str(rrc));                               \
 175 } while (0)
 176
 177 #define GNILND_API_BUSY(args...)                                                \
 178 do {                                                                            \
 179         CERROR("resources busy for "apick_fn"("apick_fmt") rc %s\n",            \
 180                 ##args, kgnilnd_api_rc2str(rrc));                               \
 181 } while (0)
 182
 183 #undef DEBUG_SMSG_CREDITS
 184 #ifdef DEBUG_SMSG_CREDITS
 185 #define CRAY_CONFIG_GHAL_GEMINI
 186 #include <gni_priv.h>
 187 #define GNIDBG_SMSG_CREDS(level, conn)                                        \
 188 do {                                                                          \
 189         gni_ep_smsg_mbox_t *smsg = conn->gnc_ephandle->smsg;                  \
 190         CDEBUG(level, "SMSGDBG: conn %p mcred %d/%d bcred %d/%d "             \
 191                 "s_seq %d/%d/%d r_seq %d/%d/%d retr %d\n",                    \
 192                 conn, smsg->mbox_credits, smsg->back_mbox_credits,            \
 193                 smsg->buffer_credits, smsg->back_buffer_credits,              \
 194                 smsg->s_seqno, smsg->s_seqno_back_mbox_credits,               \
 195                 smsg->s_seqno_back_buffer_credits, smsg->r_seqno,             \
 196                 smsg->r_seqno_back_mbox_credits,                              \
 197                 smsg->r_seqno_back_buffer_credits, smsg->retransmit_count);   \
 198 } while (0)
 199 #else
 200 #define GNIDBG_SMSG_CREDS(level, conn) do {} while(0)
 201 #endif
 202
 203 /* these are all wrappers around gni_XXX functions.
 204  * This allows us to handle all the return codes and api checks without
 205  * dirtying up the logic code */
 206
 207 /* TODO: RETURN wrapper that translates integer to GNI API RC string */
 208
 209 #define apick_fn "kgnilnd_cdm_create"
 210 #define apick_fmt "%u, %u, %u, %u, 0x%p"
 211 static inline gni_return_t kgnilnd_cdm_create(
 212                 IN uint32_t             inst_id,
 213                 IN uint8_t              ptag,
 214                 IN uint32_t             cookie,
 215                 IN uint32_t             modes,
 216                 OUT gni_cdm_handle_t    *cdm_hndl
 217                 )
 218 {
 219         gni_return_t rrc;
 220
 221         /* Error injection */
 222         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CDM_CREATE)) {
 223                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 224         } else {
 225                 rrc = gni_cdm_create(inst_id, ptag, cookie, modes, cdm_hndl);
 226         }
 227
 228         switch (rrc)  {
 229         case GNI_RC_SUCCESS:
 230                 break;
 231         case GNI_RC_ERROR_RESOURCE:
 232         case GNI_RC_INVALID_PARAM:
 233                 /* Try to bail gracefully */
 234                 GNILND_API_SWBUG(
 235                         inst_id, ptag, cookie, modes, cdm_hndl);
 236                 break;
 237         default:
 238                 GNILND_API_RC_LBUG(
 239                         inst_id, ptag, cookie, modes, cdm_hndl);
 240
 241                 /* LBUG never returns, but just for style and consistency */
 242                 break;
 243         }
 244         RETURN(rrc);
 245 }
 246
 247 #undef apick_fn
 248 #undef apick_fmt
 249
 250 #define apick_fn "kgnilnd_cdm_attach"
 251 #define apick_fmt "0x%p, %u, 0x%p, 0x%p"
 252 static inline gni_return_t kgnilnd_cdm_attach(
 253                 IN gni_cdm_handle_t     cdm_hndl,
 254                 IN uint32_t             device_id,
 255                 OUT uint32_t            *local_addr,
 256                 OUT gni_nic_handle_t    *nic_hndl
 257                 )
 258 {
 259         gni_return_t rrc;
 260
 261         /* Error injection */
 262         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CDM_ATTACH)) {
 263                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 264         } else {
 265                 rrc = gni_cdm_attach(cdm_hndl, device_id, local_addr, nic_hndl);
 266         }
 267
 268         switch (rrc)  {
 269         case GNI_RC_SUCCESS:
 270                 break;
 271         case GNI_RC_NO_MATCH:
 272         case GNI_RC_INVALID_PARAM:
 273                 GNILND_API_SWBUG(
 274                         cdm_hndl, device_id, local_addr, nic_hndl);
 275                 break;
 276         case GNI_RC_ERROR_RESOURCE:
 277         case GNI_RC_INVALID_STATE:
 278                 GNILND_API_RESOURCE(
 279                         cdm_hndl, device_id, local_addr, nic_hndl);
 280                 break;
 281         default:
 282                 GNILND_API_RC_LBUG(
 283                         cdm_hndl, device_id, local_addr, nic_hndl);
 284
 285                 /* LBUG never returns, but just for style and consistency */
 286                 break;
 287         }
 288         RETURN(rrc);
 289 }
 290 #undef apick_fmt
 291 #undef apick_fn
 292
 293 #define apick_fn "kgnilnd_cdm_destroy"
 294 #define apick_fmt "0x%p"
 295 static inline gni_return_t kgnilnd_cdm_destroy(
 296                 IN gni_cdm_handle_t     cdm_hndl
 297                 )
 298 {
 299         gni_return_t rrc;
 300
 301         /* Error injection */
 302         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_DESTROY)) {
 303                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 304         } else {
 305                 rrc = gni_cdm_destroy(
 306                         cdm_hndl);
 307         }
 308
 309         switch (rrc)  {
 310         case GNI_RC_SUCCESS:
 311                 break;
 312         case GNI_RC_INVALID_PARAM:
 313                 GNILND_API_SWBUG(
 314                         cdm_hndl);
 315                 break;
 316         default:
 317                 GNILND_API_RC_LBUG(
 318                         cdm_hndl);
 319
 320                 /* LBUG never returns, but just for style and consistency */
 321                 break;
 322         }
 323         RETURN(rrc);
 324 }
 325 #undef apick_fn
 326 #undef apick_fmt
 327
 328 #define apick_fn "kgnilnd_subscribe_errors"
 329 #define apick_fmt "0x%p,%x,%u,0x%p,0x%p,0x%p"
 330 static inline gni_return_t kgnilnd_subscribe_errors(
 331                 IN gni_nic_handle_t  nic_handle,
 332                 IN gni_error_mask_t  mask,
 333                 IN uint32_t          EEQ_size,
 334                 IN void              (*EQ_new_event)(gni_err_handle_t),
 335                 IN void              (*app_crit_err)(gni_err_handle_t),
 336                 OUT gni_err_handle_t *err_handle
 337                 )
 338 {
 339         gni_return_t rrc;
 340
 341         /* Error injection */
 342         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_ERR_SUBSCRIBE)) {
 343                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 344         } else {
 345                 rrc = gni_subscribe_errors(
 346                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 347                         err_handle);
 348         }
 349
 350         switch (rrc)  {
 351         case GNI_RC_SUCCESS:
 352                 break;
 353         case GNI_RC_INVALID_PARAM:
 354                 GNILND_API_SWBUG(
 355                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 356                         err_handle);
 357                 break;
 358         case GNI_RC_ERROR_RESOURCE:
 359                 GNILND_API_RESOURCE(
 360                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 361                         err_handle);
 362                 break;
 363         default:
 364                 GNILND_API_RC_LBUG(
 365                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 366                         err_handle);
 367                 /* LBUG never returns, but just for style and consistency */
 368                 break;
 369         }
 370         RETURN(rrc);
 371 }
 372 #undef apick_fn
 373 #undef apick_fmt
 374
 375 #define apick_fn "kgnilnd_release_errors"
 376 #define apick_fmt "0x%p"
 377 static inline gni_return_t kgnilnd_release_errors(
 378                 IN gni_err_handle_t err_handle
 379                 )
 380 {
 381         gni_return_t rrc;
 382
 383         rrc = gni_release_errors(
 384                         err_handle);
 385
 386         switch (rrc)  {
 387         case GNI_RC_SUCCESS:
 388                 break;
 389         case GNI_RC_INVALID_PARAM:
 390         case GNI_RC_NOT_DONE:
 391                 GNILND_API_SWBUG(
 392                         err_handle);
 393                 break;
 394         default:
 395                 GNILND_API_RC_LBUG(
 396                         err_handle);
 397                 /* LBUG never returns, but just for style and consistency */
 398                 break;
 399         }
 400         RETURN(rrc);
 401 }
 402 #undef apick_fn
 403 #undef apick_fmt
 404
 405 #define apick_fn "kgnilnd_set_quiesce_callback"
 406 #define apick_fmt "0x%p,0x%p"
 407 static inline gni_return_t kgnilnd_set_quiesce_callback(
 408                 IN gni_nic_handle_t  nic_handle,
 409                 IN void              (*qsce_func)(gni_nic_handle_t, uint64_t msecs)
 410                 )
 411 {
 412         gni_return_t rrc;
 413
 414         /* Error injection */
 415         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_REG_QUIESCE)) {
 416                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 417         } else {
 418                 rrc = gni_set_quiesce_callback(
 419                         nic_handle, qsce_func);
 420         }
 421
 422         switch (rrc)  {
 423         case GNI_RC_SUCCESS:
 424                 break;
 425         case GNI_RC_INVALID_STATE:
 426         case GNI_RC_INVALID_PARAM:
 427                 GNILND_API_SWBUG(
 428                         nic_handle, qsce_func);
 429                 break;
 430         default:
 431                 GNILND_API_RC_LBUG(
 432                         nic_handle, qsce_func);
 433                 /* LBUG never returns, but just for style and consistency */
 434                 break;
 435         }
 436         RETURN(rrc);
 437 }
 438 #undef apick_fn
 439 #undef apick_fmt
 440
 441 #define apick_fn "kgnilnd_get_quiesce_status"
 442 #define apick_fmt "0x%p"
 443 static inline gni_return_t kgnilnd_get_quiesce_status(
 444                 IN gni_nic_handle_t  nic_handle
 445                 )
 446 {
 447         uint32_t rrc;
 448
 449         /* this has weird RC -
 450          * 0 - quiesce not in progress
 451          * 1 - quiesce is turned on
 452         */
 453
 454         /* Error injection */
 455         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_IN_QUIESCE)) {
 456                 rrc = 1;
 457         } else {
 458                 rrc = gni_get_quiesce_status(
 459                         nic_handle);
 460         }
 461
 462         switch (rrc)  {
 463         case 1:
 464         case 0:
 465                 break;
 466         default:
 467                 GNILND_API_RC_LBUG(
 468                         nic_handle);
 469                 /* LBUG never returns, but just for style and consistency */
 470                 break;
 471         }
 472         RETURN(rrc);
 473 }
 474 #undef apick_fn
 475 #undef apick_fmt
 476
 477 #define apick_fn "kgnilnd_cq_create"
 478 #define apick_fmt "0x%p, %u, %u, 0x%p, "LPX64", 0x%p"
 479 static inline gni_return_t kgnilnd_cq_create(
 480                 IN gni_nic_handle_t     nic_hndl,
 481                 IN uint32_t             entry_count,
 482                 IN uint32_t             delay_index,
 483                 IN gni_cq_event_hndlr_f *event_handler,
 484                 IN uint64_t             usr_event_data,
 485                 OUT gni_cq_handle_t     *cq_hndl
 486                 )
 487 {
 488         gni_return_t rrc;
 489
 490         /* Error injection */
 491         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_CREATE)) {
 492                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 493         } else {
 494                 rrc = gni_cq_create(
 495                        nic_hndl, entry_count, delay_index, event_handler,
 496                         usr_event_data, cq_hndl);
 497         }
 498
 499         switch (rrc)  {
 500         case GNI_RC_SUCCESS:
 501                 break;
 502         case GNI_RC_INVALID_PARAM:
 503                 GNILND_API_SWBUG(
 504                         nic_hndl, entry_count, delay_index, event_handler,
 505                         usr_event_data, cq_hndl);
 506                 break;
 507         case GNI_RC_ERROR_RESOURCE:
 508                 GNILND_API_RESOURCE(
 509                         nic_hndl, entry_count, delay_index, event_handler,
 510                         usr_event_data, cq_hndl);
 511                 break;
 512         default:
 513                 GNILND_API_RC_LBUG(
 514                         nic_hndl, entry_count, delay_index, event_handler,
 515                         usr_event_data, cq_hndl);
 516
 517                 /* LBUG never returns, but just for style and consistency */
 518                 break;
 519         }
 520         RETURN(rrc);
 521 }
 522 #undef apick_fn
 523 #undef apick_fmt
 524
 525 #define apick_fn "kgnilnd_cq_destroy"
 526 #define apick_fmt "0x%p"
 527 static inline gni_return_t kgnilnd_cq_destroy(
 528                 IN gni_cq_handle_t cq_hndl
 529                 )
 530 {
 531         gni_return_t rrc;
 532
 533         /* Error injection */
 534         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_DESTROY)) {
 535                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 536         } else {
 537
 538                 rrc = gni_cq_destroy(
 539                         cq_hndl);
 540         }
 541
 542         switch (rrc)  {
 543         case GNI_RC_SUCCESS:
 544                 break;
 545         case GNI_RC_INVALID_PARAM:
 546                 GNILND_API_SWBUG(
 547                         cq_hndl);
 548                 break;
 549         case GNI_RC_ERROR_RESOURCE:
 550                 GNILND_API_BUSY(
 551                         cq_hndl);
 552                 break;
 553         default:
 554                 GNILND_API_RC_LBUG(
 555                         cq_hndl);
 556
 557                 /* LBUG never returns, but just for style and consistency */
 558                 break;
 559         }
 560         RETURN(rrc);
 561 }
 562 #undef apick_fn
 563 #undef apick_fmt
 564
 565 #define apick_fn "kgnilnd_cq_get_event"
 566 #define apick_fmt "0x%p, 0x%p"
 567 static inline gni_return_t kgnilnd_cq_get_event(
 568                 IN gni_cq_handle_t cq_hndl,
 569                 OUT gni_cq_entry_t *event_data
 570                 )
 571 {
 572         gni_return_t rrc;
 573
 574         /* no error injection - CQs are touchy about the data.
 575          * where appropriate, we'll do this on the CQs that should be able to
 576          * handle the various errors */
 577         rrc = gni_cq_get_event(
 578                         cq_hndl, event_data);
 579
 580         switch (rrc)  {
 581         case GNI_RC_SUCCESS:
 582         case GNI_RC_NOT_DONE:
 583         case GNI_RC_TRANSACTION_ERROR:
 584                 break;
 585         case GNI_RC_ERROR_RESOURCE:
 586                 LASSERTF(GNI_CQ_OVERRUN(*event_data),
 587                          "kgni returned ERROR_RESOURCE but cq_hndl 0x%p is not "
 588                          "overrun\n", cq_hndl);
 589                 break;
 590         case GNI_RC_INVALID_PARAM:
 591                 GNILND_API_SWBUG(
 592                         cq_hndl, event_data);
 593                 break;
 594         default:
 595                 GNILND_API_RC_LBUG(
 596                         cq_hndl, event_data);
 597
 598                 /* LBUG never returns, but just for style and consistency */
 599                 break;
 600         }
 601         return rrc;
 602 }
 603 #undef apick_fn
 604 #undef apick_fmt
 605
 606 #define apick_fn "kgnilnd_smsg_init"
 607 #define apick_fmt "0x%p, 0x%p, 0x%p"
 608 static inline gni_return_t kgnilnd_smsg_init(
 609                 IN gni_ep_handle_t      ep_hndl,
 610                 IN gni_smsg_attr_t      *local_smsg_attr,
 611                 IN gni_smsg_attr_t      *remote_smsg_attr
 612                 )
 613 {
 614         gni_return_t rrc;
 615
 616         /* Error injection */
 617         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_INIT)) {
 618                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_RESOURCE;
 619         } else {
 620                 rrc = gni_smsg_init(
 621                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 622         }
 623
 624         switch (rrc)  {
 625         /* both of these are OK, upper SW needs to handle */
 626         case GNI_RC_SUCCESS:
 627         case GNI_RC_NOT_DONE:
 628                 break;
 629         case GNI_RC_INVALID_PARAM:
 630         case GNI_RC_INVALID_STATE:
 631                 GNILND_API_SWBUG(
 632                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 633                 break;
 634         case GNI_RC_ERROR_RESOURCE:
 635                 GNILND_API_RESOURCE(
 636                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 637                 break;
 638         default:
 639                 GNILND_API_RC_LBUG(
 640                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 641
 642                 /* LBUG never returns, but just for style and consistency */
 643                 break;
 644         }
 645         RETURN(rrc);
 646 }
 647 #undef apick_fn
 648 #undef apick_fmt
 649
 650 #define apick_fn "kgnilnd_smsg_send"
 651 #define apick_fmt "0x%p, 0x%p, %d, 0x%p, %u %u"
 652 static inline gni_return_t kgnilnd_smsg_send(
 653                 IN gni_ep_handle_t      ep_hndl,
 654                 IN void                 *header,
 655                 IN uint32_t             header_length,
 656                 IN void                 *data,
 657                 IN uint32_t             data_length,
 658                 IN uint32_t             msg_id
 659                 )
 660 {
 661         gni_return_t rrc;
 662
 663         /* Error injection */
 664         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_SEND)) {
 665                 if (cfs_fail_loc & CFS_FAIL_RAND) {
 666                         rrc = GNI_RC_NOT_DONE;
 667                 } else {
 668                         rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 669                 }
 670         } else {
 671                 rrc = gni_smsg_send(
 672                         ep_hndl, header, header_length, data, data_length, msg_id);
 673         }
 674
 675         switch (rrc)  {
 676         /* both of these are OK, upper SW needs to handle */
 677         case GNI_RC_SUCCESS:
 678         case GNI_RC_NOT_DONE:
 679                 break;
 680         case GNI_RC_INVALID_PARAM:
 681                 GNILND_API_SWBUG(
 682                         ep_hndl, header, header_length, data, data_length, msg_id);
 683                 break;
 684         case GNI_RC_ERROR_RESOURCE:
 685                 GNILND_API_RESOURCE(
 686                         ep_hndl, header, header_length, data, data_length, msg_id);
 687                 break;
 688         default:
 689                 GNILND_API_RC_LBUG(
 690                         ep_hndl, header, header_length, data, data_length, msg_id);
 691
 692                 /* LBUG never returns, but just for style and consistency */
 693                 break;
 694         }
 695         RETURN(rrc);
 696 }
 697 #undef apick_fn
 698 #undef apick_fmt
 699
 700 #define apick_fn "kgnilnd_smsg_getnext"
 701 #define apick_fmt "0x%p,0x%p"
 702 static inline gni_return_t kgnilnd_smsg_getnext(
 703                 IN gni_ep_handle_t      ep_hndl,
 704                 OUT void                **header
 705                 )
 706 {
 707         gni_return_t rrc;
 708
 709         /* Error injection */
 710         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_RELEASE)) {
 711                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 712         } else {
 713                 rrc = gni_smsg_getnext(
 714                         ep_hndl, header);
 715         }
 716
 717         switch (rrc)  {
 718         /* both of these are OK, upper SW needs to handle */
 719         case GNI_RC_SUCCESS:
 720         case GNI_RC_NOT_DONE:
 721         case GNI_RC_INVALID_STATE:
 722                 break;
 723         case GNI_RC_INVALID_PARAM:
 724                 GNILND_API_SWBUG(
 725                         ep_hndl, header);
 726                 break;
 727         default:
 728                 GNILND_API_RC_LBUG(
 729                         ep_hndl, header);
 730
 731                 /* LBUG never returns, but just for style and consistency */
 732                 break;
 733         }
 734         RETURN(rrc);
 735 }
 736 #undef apick_fn
 737 #undef apick_fmt
 738
 739 #define apick_fn "kgnilnd_smsg_release"
 740 #define apick_fmt "0x%p"
 741 static inline gni_return_t kgnilnd_smsg_release(
 742                 IN gni_ep_handle_t      ep_hndl
 743                 )
 744 {
 745         gni_return_t rrc;
 746
 747         /* Error injection */
 748         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_RELEASE)) {
 749                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 750         } else {
 751                 rrc = gni_smsg_release(
 752                         ep_hndl);
 753         }
 754
 755         switch (rrc)  {
 756         /* both of these are OK, upper SW needs to handle */
 757         case GNI_RC_SUCCESS:
 758         case GNI_RC_NOT_DONE:
 759                 break;
 760         case GNI_RC_INVALID_PARAM:
 761                 GNILND_API_SWBUG(
 762                         ep_hndl);
 763                 break;
 764         default:
 765                 GNILND_API_RC_LBUG(
 766                         ep_hndl);
 767
 768                 /* LBUG never returns, but just for style and consistency */
 769                 break;
 770         }
 771         RETURN(rrc);
 772 }
 773 #undef apick_fn
 774 #undef apick_fmt
 775
 776 #define apick_fn "kgnilnd_ep_create"
 777 #define apick_fmt "0x%p, 0x%p, 0x%p"
 778 static inline gni_return_t kgnilnd_ep_create(
 779                 IN gni_nic_handle_t     nic_hndl,
 780                 IN gni_cq_handle_t      src_cq_hndl,
 781                 OUT gni_ep_handle_t     *ep_hndl
 782                 )
 783 {
 784         gni_return_t rrc;
 785
 786         /* error injection */
 787         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_CREATE)) {
 788                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_NOMEM;
 789         } else {
 790                 rrc = gni_ep_create(
 791                         nic_hndl, src_cq_hndl, ep_hndl);
 792         }
 793
 794         switch (rrc)  {
 795         case GNI_RC_SUCCESS:
 796                 break;
 797         case GNI_RC_INVALID_PARAM:
 798                 GNILND_API_SWBUG(
 799                         nic_hndl, src_cq_hndl, ep_hndl);
 800                 break;
 801         case GNI_RC_ERROR_NOMEM:
 802                 GNILND_API_RESOURCE(
 803                         nic_hndl, src_cq_hndl, ep_hndl);
 804                 break;
 805         default:
 806                 GNILND_API_RC_LBUG(
 807                         nic_hndl, src_cq_hndl, ep_hndl);
 808
 809                 /* lbug never returns, but just for style and consistency */
 810                 break;
 811         }
 812         RETURN(rrc);
 813 }
 814 #undef apick_fn
 815 #undef apick_fmt
 816
 817 #define apick_fn "kgnilnd_ep_bind"
 818 #define apick_fmt "0x%p, %x, %x"
 819 static inline gni_return_t kgnilnd_ep_bind(
 820                 IN gni_ep_handle_t      ep_hndl,
 821                 IN uint32_t             remote_addr,
 822                 IN uint32_t             remote_id
 823                 )
 824 {
 825         gni_return_t rrc;
 826
 827         /* error injection */
 828         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_BIND)) {
 829                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 830         } else {
 831                 rrc = gni_ep_bind(
 832                         ep_hndl, remote_addr, remote_id);
 833         }
 834
 835         switch (rrc)  {
 836         /* both of these are ok, upper sw needs to handle */
 837         case GNI_RC_SUCCESS:
 838         case GNI_RC_NOT_DONE:
 839                 break;
 840         case GNI_RC_INVALID_PARAM:
 841                 GNILND_API_SWBUG(
 842                         ep_hndl, remote_addr, remote_id);
 843                 break;
 844         default:
 845                 GNILND_API_RC_LBUG(
 846                         ep_hndl, remote_addr, remote_id);
 847
 848                 /* lbug never returns, but just for style and consistency */
 849                 break;
 850         }
 851         RETURN(rrc);
 852 }
 853 #undef apick_fn
 854 #undef apick_fmt
 855
 856 #define apick_fn "kgnilnd_ep_set_eventdata"
 857 #define apick_fmt "0x%p, %x, %x"
 858 static inline gni_return_t kgnilnd_ep_set_eventdata(
 859                 IN gni_ep_handle_t      ep_hndl,
 860                 IN uint32_t             local_event,
 861                 IN uint32_t             remote_event
 862                 )
 863 {
 864         gni_return_t rrc;
 865
 866         /* Error injection */
 867         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_SET_EVDATA)) {
 868                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 869         } else {
 870                 rrc = gni_ep_set_eventdata(
 871                         ep_hndl, local_event, remote_event);
 872         }
 873
 874         switch (rrc)  {
 875         case GNI_RC_SUCCESS:
 876                 break;
 877         case GNI_RC_INVALID_PARAM:
 878                 GNILND_API_SWBUG(
 879                         ep_hndl, local_event, remote_event);
 880                 break;
 881         default:
 882                 GNILND_API_RC_LBUG(
 883                         ep_hndl, local_event, remote_event);
 884
 885                 /* LBUG never returns, but just for style and consistency */
 886                 break;
 887         }
 888         RETURN(rrc);
 889 }
 890 #undef apick_fn
 891 #undef apick_fmt
 892
 893 #define apick_fn "kgnilnd_ep_unbind"
 894 #define apick_fmt "0x%p"
 895 static inline gni_return_t kgnilnd_ep_unbind(
 896                 IN gni_ep_handle_t      ep_hndl
 897                 )
 898 {
 899         gni_return_t rrc;
 900
 901         /* Error injection */
 902         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_UNBIND)) {
 903                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 904         } else {
 905                 rrc = gni_ep_unbind(
 906                         ep_hndl);
 907         }
 908
 909         switch (rrc)  {
 910         /* both of these are OK, upper SW needs to handle */
 911         case GNI_RC_NOT_DONE:
 912         case GNI_RC_SUCCESS:
 913                 break;
 914         case GNI_RC_INVALID_PARAM:
 915                 GNILND_API_SWBUG(
 916                         ep_hndl);
 917                 break;
 918         default:
 919                 GNILND_API_RC_LBUG(
 920                         ep_hndl);
 921
 922                 /* LBUG never returns, but just for style and consistency */
 923                 break;
 924         }
 925         RETURN(rrc);
 926 }
 927 #undef apick_fn
 928 #undef apick_fmt
 929
 930 #define apick_fn "kgnilnd_ep_destroy"
 931 #define apick_fmt "0x%p"
 932 static inline gni_return_t kgnilnd_ep_destroy(
 933                 IN gni_ep_handle_t      ep_hndl
 934                 )
 935 {
 936         gni_return_t rrc;
 937
 938         /* Error injection */
 939         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_DESTROY)) {
 940                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 941         } else {
 942                 rrc = gni_ep_destroy(
 943                         ep_hndl);
 944         }
 945
 946         switch (rrc)  {
 947         case GNI_RC_SUCCESS:
 948                 break;
 949         case GNI_RC_INVALID_PARAM:
 950                 GNILND_API_SWBUG(
 951                         ep_hndl);
 952                 break;
 953         default:
 954                 GNILND_API_RC_LBUG(
 955                         ep_hndl);
 956
 957                 /* LBUG never returns, but just for style and consistency */
 958                 break;
 959         }
 960         RETURN(rrc);
 961 }
 962 #undef apick_fn
 963 #undef apick_fmt
 964
 965 #define apick_fn "kgnilnd_ep_postdata_w_id"
 966 #define apick_fmt "0x%p, 0x%p, %d, 0x%p, %d, "LPU64""
 967 static inline gni_return_t kgnilnd_ep_postdata_w_id(
 968                 IN gni_ep_handle_t ep_hndl,
 969                 IN void            *in_data,
 970                 IN uint16_t        data_len,
 971                 IN void            *out_buf,
 972                 IN uint16_t        buf_size,
 973                 IN uint64_t        datagram_id
 974                 )
 975 {
 976         gni_return_t rrc;
 977
 978         /* Error injection */
 979         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_POST)) {
 980                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_SIZE_ERROR;
 981         } else {
 982                 rrc = gni_ep_postdata_w_id(
 983                         ep_hndl, in_data, data_len, out_buf, buf_size,
 984                         datagram_id);
 985         }
 986
 987         switch (rrc)  {
 988         case GNI_RC_SUCCESS:
 989         case GNI_RC_ERROR_NOMEM:
 990         case GNI_RC_ERROR_RESOURCE:
 991                 break;
 992         case GNI_RC_INVALID_PARAM:
 993         case GNI_RC_SIZE_ERROR:
 994                 GNILND_API_SWBUG(
 995                         ep_hndl, in_data, data_len, out_buf, buf_size,
 996                         datagram_id);
 997                 break;
 998         default:
 999                 GNILND_API_RC_LBUG(
1000                         ep_hndl, in_data, data_len, out_buf, buf_size,
1001                         datagram_id);
1002
1003                 /* LBUG never returns, but just for style and consistency */
1004                 break;
1005         }
1006         RETURN(rrc);
1007 }
1008 #undef apick_fn
1009 #undef apick_fmt
1010
1011 #define apick_fn "kgnilnd_ep_postdata_test_by_id"
1012 #define apick_fmt "0x%p, "LPU64", 0x%p, 0x%p, 0x%p"
1013 static inline gni_return_t kgnilnd_ep_postdata_test_by_id(
1014                 IN gni_ep_handle_t      ep_hndl,
1015                 IN uint64_t             datagram_id,
1016                 OUT gni_post_state_t    *post_state,
1017                 OUT uint32_t            *remote_addr,
1018                 OUT uint32_t            *remote_id
1019                 )
1020 {
1021         gni_return_t rrc;
1022
1023         /* Error injection */
1024         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_TEST)) {
1025                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_NOMEM;
1026         } else {
1027                 rrc = gni_ep_postdata_test_by_id(
1028                         ep_hndl, datagram_id, post_state, remote_addr,
1029                         remote_id);
1030
1031                 /* we want to lie, but we need to do the actual work first
1032                  * so we don't keep getting the event saying a dgram is ready */
1033                 if (rrc == GNI_RC_SUCCESS && CFS_FAIL_CHECK(CFS_FAIL_GNI_DG_TERMINATE)) {
1034                         /* don't use fail_val, allows us to do FAIL_SOME */
1035                         *post_state = GNI_POST_TERMINATED;
1036                 }
1037         }
1038
1039         switch (rrc)  {
1040         case GNI_RC_SUCCESS:
1041         case GNI_RC_NO_MATCH:
1042                 break;
1043         case GNI_RC_SIZE_ERROR:
1044         case GNI_RC_INVALID_PARAM:
1045                 GNILND_API_SWBUG(
1046                         ep_hndl, datagram_id, post_state, remote_addr,
1047                         remote_id);
1048                 break;
1049         case GNI_RC_ERROR_NOMEM:
1050                 GNILND_API_RESOURCE(
1051                         ep_hndl, datagram_id, post_state, remote_addr,
1052                         remote_id);
1053                 break;
1054         default:
1055                 GNILND_API_RC_LBUG(
1056                         ep_hndl, datagram_id, post_state, remote_addr,
1057                         remote_id);
1058
1059                 /* LBUG never returns, but just for style and consistency */
1060                 break;
1061         }
1062         RETURN(rrc);
1063 }
1064 #undef apick_fn
1065 #undef apick_fmt
1066
1067 #define apick_fn "kgnilnd_ep_postdata_cancel_by_id"
1068 #define apick_fmt "0x%p, "LPU64""
1069 static inline gni_return_t kgnilnd_ep_postdata_cancel_by_id(
1070                 IN gni_ep_handle_t      ep_hndl,
1071                 IN uint64_t             datagram_id
1072                 )
1073 {
1074         gni_return_t rrc;
1075
1076         /* no error injection as the only thing we'd do is LBUG */
1077
1078         rrc = gni_ep_postdata_cancel_by_id(
1079                 ep_hndl, datagram_id);
1080
1081         switch (rrc)  {
1082         case GNI_RC_SUCCESS:
1083         case GNI_RC_NO_MATCH:
1084                 break;
1085         case GNI_RC_INVALID_PARAM:
1086                 GNILND_API_SWBUG(
1087                         ep_hndl, datagram_id);
1088                 break;
1089         default:
1090                 GNILND_API_RC_LBUG(
1091                         ep_hndl, datagram_id);
1092
1093                 /* LBUG never returns, but just for style and consistency */
1094                 break;
1095         }
1096         RETURN(rrc);
1097 }
1098 #undef apick_fn
1099 #undef apick_fmt
1100
1101 #define apick_fn "kgnilnd_postdata_probe_by_id"
1102 #define apick_fmt "0x%p, 0x%p"
1103 static inline gni_return_t kgnilnd_postdata_probe_by_id(
1104                 IN gni_nic_handle_t    nic_hndl,
1105                 OUT uint64_t          *datagram_id
1106                 )
1107 {
1108         gni_return_t rrc;
1109
1110         /* Error injection */
1111         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PROBE)) {
1112                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NO_MATCH;
1113         } else {
1114                 rrc = gni_postdata_probe_by_id(
1115                         nic_hndl, datagram_id);
1116         }
1117
1118         switch (rrc)  {
1119         case GNI_RC_SUCCESS:
1120         case GNI_RC_NO_MATCH:
1121                 break;
1122         case GNI_RC_INVALID_PARAM:
1123                 GNILND_API_SWBUG(
1124                         nic_hndl, datagram_id);
1125                 break;
1126         default:
1127                 GNILND_API_RC_LBUG(
1128                         nic_hndl, datagram_id);
1129
1130                 /* LBUG never returns, but just for style and consistency */
1131                 break;
1132         }
1133         RETURN(rrc);
1134 }
1135 #undef apick_fn
1136 #undef apick_fmt
1137
1138 #define apick_fn "kgnilnd_postdata_probe_wait_by_id"
1139 #define apick_fmt "0x%p, %d, 0x%p"
1140 static inline gni_return_t kgnilnd_postdata_probe_wait_by_id(
1141                 IN gni_nic_handle_t nic_hndl,
1142                 IN uint32_t         timeout,
1143                 OUT uint64_t        *datagram_id
1144                 )
1145 {
1146         gni_return_t rrc;
1147
1148         /* Error injection */
1149         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PROBE_WAIT)) {
1150                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_TIMEOUT;
1151         } else {
1152                 rrc = gni_postdata_probe_wait_by_id(
1153                         nic_hndl, timeout, datagram_id);
1154         }
1155
1156         switch (rrc)  {
1157         case GNI_RC_SUCCESS:
1158         case GNI_RC_TIMEOUT:
1159                 break;
1160         case GNI_RC_INVALID_PARAM:
1161                 GNILND_API_SWBUG(
1162                         nic_hndl, timeout, datagram_id);
1163                 break;
1164         default:
1165                 GNILND_API_RC_LBUG(
1166                         nic_hndl, timeout, datagram_id);
1167
1168                 /* LBUG never returns, but just for style and consistency */
1169                 break;
1170         }
1171         RETURN(rrc);
1172 }
1173 #undef apick_fn
1174 #undef apick_fmt
1175
1176 #define apick_fn "kgnilnd_post_rdma"
1177 #define apick_fmt "0x%p, 0x%p"
1178 static inline gni_return_t kgnilnd_post_rdma(
1179                 IN gni_ep_handle_t               ep_hndl,
1180                 IN gni_post_descriptor_t        *post_descr
1181                 )
1182 {
1183         gni_return_t rrc;
1184
1185         /* Error injection */
1186         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_POST_RDMA)) {
1187                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
1188         } else {
1189                 rrc = gni_post_rdma(
1190                         ep_hndl, post_descr);
1191         }
1192
1193         switch (rrc)  {
1194         case GNI_RC_SUCCESS:
1195                 break;
1196         case GNI_RC_ALIGNMENT_ERROR:
1197         case GNI_RC_INVALID_PARAM:
1198                 GNILND_API_SWBUG(
1199                         ep_hndl, post_descr);
1200                 break;
1201         case GNI_RC_ERROR_RESOURCE:
1202                 GNILND_API_RESOURCE(
1203                         ep_hndl, post_descr);
1204                 break;
1205         default:
1206                 GNILND_API_RC_LBUG(
1207                         ep_hndl, post_descr);
1208
1209                 /* LBUG never returns, but just for style and consistency */
1210                 break;
1211         }
1212         RETURN(rrc);
1213 }
1214 #undef apick_fn
1215 #undef apick_fmt
1216
1217 #define apick_fn "kgnilnd_get_completed"
1218 #define apick_fmt "0x%p,"LPX64",0x%p"
1219 static inline gni_return_t kgnilnd_get_completed(
1220                 IN gni_cq_handle_t              cq_hndl,
1221                 IN gni_cq_entry_t               event_data,
1222                 OUT gni_post_descriptor_t       **post_descr
1223                 )
1224 {
1225         gni_return_t rrc;
1226
1227
1228         rrc = gni_get_completed(cq_hndl, event_data, post_descr);
1229
1230         switch (rrc)  {
1231         case GNI_RC_TRANSACTION_ERROR:
1232         case GNI_RC_SUCCESS:
1233                 break;
1234         case GNI_RC_DESCRIPTOR_ERROR:
1235         case GNI_RC_INVALID_PARAM:
1236                 GNILND_API_SWBUG(cq_hndl, event_data, post_descr);
1237                 break;
1238         default:
1239                 GNILND_API_RC_LBUG(cq_hndl, event_data, post_descr);
1240                 /* LBUG never returns, but just for style and consistency */
1241                 break;
1242         }
1243
1244         /* Error injection - we need a valid desc, so let kgni give us one
1245          * - then we lie  */
1246         if (rrc == GNI_RC_SUCCESS &&
1247             (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED))) {
1248                 /* We only trigger TRANSACTION_ERROR for now */
1249                 gni_post_descriptor_t *desc;
1250                 rrc = GNI_RC_TRANSACTION_ERROR;
1251                 desc = *post_descr;
1252                 desc->status = rrc;
1253                 /* recoverable decision made from cfs_fail_val in
1254                  *  kgnilnd_cq_error_str and
1255                  *  kgnilnd_cq_error_recoverable */
1256         }
1257         RETURN(rrc);
1258 }
1259 #undef apick_fn
1260 #undef apick_fmt
1261
1262 #define apick_fn "kgnilnd_cq_error_str"
1263 #define apick_fmt LPX64",0x%p,%d"
1264 static inline gni_return_t kgnilnd_cq_error_str(
1265                 IN gni_cq_entry_t       entry,
1266                 IN void                *buffer,
1267                 IN uint32_t             len
1268                 )
1269 {
1270         gni_return_t rrc;
1271
1272         /* Error injection - set string if we injected a
1273          *  TRANSACTION_ERROR earlier */
1274         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED)) {
1275                 /* if we just set persistent error, we can't ever
1276                  * break in via ssh to clear, so use a count > 10 to indicate fatal */
1277                 sprintf(buffer, "INJECT:%s", cfs_fail_val > 10 ?
1278                         "FATAL" : "RECOVERABLE");
1279                 rrc = GNI_RC_SUCCESS;
1280         } else {
1281                 rrc = gni_cq_error_str(
1282                         entry, buffer, len);
1283         }
1284
1285         switch (rrc)  {
1286         case GNI_RC_SUCCESS:
1287                 break;
1288         case GNI_RC_SIZE_ERROR:
1289         case GNI_RC_INVALID_PARAM:
1290                 GNILND_API_SWBUG(
1291                         entry, buffer, len);
1292                 /* give them something to use */
1293                 snprintf(buffer, len, "UNDEF:UNDEF");
1294                 break;
1295         default:
1296                 GNILND_API_RC_LBUG(
1297                         entry, buffer, len);
1298
1299                 /* LBUG never returns, but just for style and consistency */
1300                 break;
1301         }
1302         RETURN(rrc);
1303 }
1304 #undef apick_fn
1305 #undef apick_fmt
1306
1307 #define apick_fn "kgnilnd_cq_error_recoverable"
1308 #define apick_fmt LPX64",0x%p"
1309 static inline gni_return_t kgnilnd_cq_error_recoverable(
1310                 IN gni_cq_entry_t       entry,
1311                 IN uint32_t            *recoverable
1312                 )
1313 {
1314         gni_return_t rrc;
1315
1316         /* Error injection - set string if we injected a
1317          *  TRANSACTION_ERROR earlier */
1318         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED)) {
1319                 *recoverable = cfs_fail_val > 10 ? 0 : 1;
1320                 rrc = GNI_RC_SUCCESS;
1321         } else {
1322                 rrc = gni_cq_error_recoverable(
1323                         entry, recoverable);
1324         }
1325
1326         switch (rrc)  {
1327         case GNI_RC_SUCCESS:
1328                 break;
1329         case GNI_RC_INVALID_STATE:
1330         case GNI_RC_INVALID_PARAM:
1331                 GNILND_API_SWBUG(
1332                         entry, recoverable);
1333                 *recoverable = 0;
1334                 break;
1335         default:
1336                 GNILND_API_RC_LBUG(
1337                         entry, recoverable);
1338
1339                 /* LBUG never returns, but just for style and consistency */
1340                 break;
1341         }
1342         RETURN(rrc);
1343 }
1344 #undef apick_fn
1345 #undef apick_fmt
1346
1347 #define apick_fn "kgnilnd_mem_register_segments"
1348 #define apick_fmt "0x%p,0x%p,%u,0x%p,%x,0x%p"
1349 static inline gni_return_t
1350 kgnilnd_mem_register_segments(
1351                 IN gni_nic_handle_t     nic_hndl,
1352                 IN gni_mem_segment_t    *mem_segments,
1353                 IN uint32_t             segments_cnt,
1354                 IN gni_cq_handle_t      dst_cq_hndl,
1355                 IN uint32_t             flags,
1356                 OUT gni_mem_handle_t    *mem_hndl
1357                 )
1358 {
1359         gni_return_t rrc;
1360
1361         /* Error injection */
1362         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PHYS_MAP)) {
1363                 rrc = GNI_RC_ERROR_RESOURCE;
1364         } else {
1365                 rrc = gni_mem_register_segments(
1366                         nic_hndl, mem_segments, segments_cnt,
1367                         dst_cq_hndl, flags, mem_hndl);
1368         }
1369
1370         switch (rrc)  {
1371         case GNI_RC_SUCCESS:
1372         case GNI_RC_ERROR_RESOURCE:
1373                 break;
1374         case GNI_RC_INVALID_PARAM:
1375                 GNILND_API_SWBUG(
1376                         nic_hndl, mem_segments, segments_cnt,
1377                         dst_cq_hndl, flags, mem_hndl);
1378                 break;
1379         default:
1380                 GNILND_API_RC_LBUG(
1381                         nic_hndl, mem_segments, segments_cnt,
1382                         dst_cq_hndl, flags, mem_hndl);
1383
1384                 /* LBUG never returns, but just for style and consistency */
1385                 break;
1386         }
1387         RETURN(rrc);
1388 }
1389 #undef apick_fn
1390 #undef apick_fmt
1391
1392 #define apick_fn "kgnilnd_mem_register"
1393 #define apick_fmt "0x%p,"LPX64","LPX64"0x%p,%u,0x%p"
1394 static inline gni_return_t kgnilnd_mem_register(
1395                 IN gni_nic_handle_t     nic_hndl,
1396                 IN uint64_t             address,
1397                 IN uint64_t             length,
1398                 IN gni_cq_handle_t      dst_cq_hndl,
1399                 IN uint32_t             flags,
1400                 OUT gni_mem_handle_t    *mem_hndl
1401                 )
1402 {
1403         gni_return_t rrc;
1404
1405         /* Error injection */
1406         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_MAP)) {
1407                 rrc = GNI_RC_ERROR_RESOURCE;
1408         } else if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_SMALL_MAP) &&
1409                    length <= *kgnilnd_tunables.kgn_max_immediate) {
1410                 rrc = GNI_RC_INVALID_PARAM;
1411         } else {
1412                 rrc = gni_mem_register(
1413                         nic_hndl, address, length,
1414                         dst_cq_hndl, flags, mem_hndl);
1415         }
1416
1417         switch (rrc)  {
1418         case GNI_RC_SUCCESS:
1419         case GNI_RC_ERROR_RESOURCE:
1420                 break;
1421         case GNI_RC_INVALID_PARAM:
1422                 GNILND_API_SWBUG(
1423                         nic_hndl, address, length,
1424                         dst_cq_hndl, flags, mem_hndl);
1425                 break;
1426         default:
1427                 GNILND_API_RC_LBUG(
1428                         nic_hndl, address, length,
1429                         dst_cq_hndl, flags, mem_hndl);
1430
1431                 /* LBUG never returns, but just for style and consistency */
1432                 break;
1433         }
1434         RETURN(rrc);
1435 }
1436 #undef apick_fn
1437 #undef apick_fmt
1438
1439 #define apick_fn "kgnilnd_mem_deregister"
1440 #define apick_fmt "0x%p,0x%p,%d"
1441 static inline gni_return_t kgnilnd_mem_deregister(
1442                 IN gni_nic_handle_t     nic_hndl,
1443                 IN gni_mem_handle_t     *mem_hndl,
1444                 IN int                  hold_timeout
1445                 )
1446 {
1447         gni_return_t rrc;
1448
1449         /* Error injection */
1450         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_UNMAP)) {
1451                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
1452         } else {
1453                 rrc = gni_mem_deregister(
1454                         nic_hndl, mem_hndl, hold_timeout);
1455         }
1456
1457         switch (rrc)  {
1458         case GNI_RC_SUCCESS:
1459                 break;
1460         case GNI_RC_INVALID_PARAM:
1461                 GNILND_API_SWBUG(
1462                         nic_hndl, mem_hndl, hold_timeout);
1463                 break;
1464         default:
1465                 GNILND_API_RC_LBUG(
1466                         nic_hndl, mem_hndl, hold_timeout);
1467
1468                 /* LBUG never returns, but just for style and consistency */
1469                 break;
1470         }
1471         RETURN(rrc);
1472 }
1473 #undef apick_fn
1474 #undef apick_fmt
1475
1476 #define apick_fn "kgnilnd_mem_mdd_release"
1477 #define apick_fmt "0x%p,0x%p"
1478 static inline gni_return_t kgnilnd_mem_mdd_release(
1479                 IN gni_nic_handle_t     nic_hndl,
1480                 IN gni_mem_handle_t     *mem_hndl
1481                 )
1482 {
1483         gni_return_t rrc;
1484
1485         /* Error injection */
1486         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_MDD_RELEASE)) {
1487                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NO_MATCH;
1488         } else {
1489                 rrc = gni_mem_mdd_release(
1490                         nic_hndl, mem_hndl);
1491         }
1492
1493         switch (rrc)  {
1494         case GNI_RC_SUCCESS:
1495         case GNI_RC_NO_MATCH:
1496                 break;
1497         default:
1498                 GNILND_API_RC_LBUG(
1499                         nic_hndl, mem_hndl);
1500
1501                 /* LBUG never returns, but just for style and consistency */
1502                 break;
1503         }
1504         RETURN(rrc);
1505 }
1506 #undef apick_fn
1507 #undef apick_fmt
1508
1509 #endif /* _GNILND_API_WRAP_H */