lnet/klnds/gnilnd/gnilnd_api_wrap.h

   1 /*
   2  * Copyright (C) 2009-2012 Cray, Inc.
   3  *
   4  *   Author: Nic Henke <nic@cray.com>
   5  *
   6  *   This file is part of Lustre, http://www.lustre.org.
   7  *
   8  *   Lustre is free software; you can redistribute it and/or
   9  *   modify it under the terms of version 2 of the GNU General Public
  10  *   License as published by the Free Software Foundation.
  11  *
  12  *   Lustre is distributed in the hope that it will be useful,
  13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *   GNU General Public License for more details.
  16  *
  17  *   You should have received a copy of the GNU General Public License
  18  *   along with Lustre; if not, write to the Free Software
  19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20  *
  21  */
  22 #ifndef _GNILND_API_WRAP_H
  23 #define _GNILND_API_WRAP_H
  24
  25 /* LNet is allocated failure locations 0xe000 to 0xffff */
  26
  27 /* GNILND has 0xf0XX */
  28 #define CFS_FAIL_GNI                    0xf000
  29 #define CFS_FAIL_GNI_PHYS_MAP           0xf001
  30 #define CFS_FAIL_GNI_VIRT_MAP           0xf002
  31 #define CFS_FAIL_GNI_GET_UNMAP          0xf003
  32 #define CFS_FAIL_GNI_PUT_UNMAP          0xf004
  33 #define CFS_FAIL_GNI_MAP_TX             0xf005
  34 #define CFS_FAIL_GNI_SMSG_SEND          0xf006
  35 #define CFS_FAIL_GNI_CLOSE_SEND         0xf007
  36 #define CFS_FAIL_GNI_CDM_CREATE         0xf008
  37 #define CFS_FAIL_GNI_CDM_DESTROY        0xf009
  38 #define CFS_FAIL_GNI_CDM_ATTACH         0xf00a
  39 #define CFS_FAIL_GNI_CQ_CREATE          0xf00b
  40 #define CFS_FAIL_GNI_CQ_DESTROY         0xf00c
  41 #define CFS_FAIL_GNI_EP_BIND            0xf00d
  42 #define CFS_FAIL_GNI_EP_UNBIND          0xf00e
  43 #define CFS_FAIL_GNI_EP_SET_EVDATA      0xf00f
  44 #define CFS_FAIL_GNI_SMSG_INIT          0xf010
  45 #define CFS_FAIL_GNI_SMSG_RELEASE       0xf011
  46 #define CFS_FAIL_GNI_POST_RDMA          0xf012
  47 #define CFS_FAIL_GNI_GET_COMPLETED      0xf013
  48 #define CFS_FAIL_GNI_EP_DESTROY         0xf015
  49 #define CFS_FAIL_GNI_VIRT_UNMAP         0xf016
  50 #define CFS_FAIL_GNI_MDD_RELEASE        0xf017
  51 #define CFS_FAIL_GNI_NOOP_SEND          0xf018
  52 #define CFS_FAIL_GNI_ERR_SUBSCRIBE      0xf01a
  53 #define CFS_FAIL_GNI_QUIESCE_RACE       0xf01b
  54 #define CFS_FAIL_GNI_DG_TERMINATE       0xf01c
  55 #define CFS_FAIL_GNI_REG_QUIESCE        0xf01d
  56 #define CFS_FAIL_GNI_IN_QUIESCE         0xf01e
  57 #define CFS_FAIL_GNI_DELAY_RDMA         0xf01f
  58 #define CFS_FAIL_GNI_SR_DOWN_RACE       0xf020
  59 #define CFS_FAIL_GNI_ALLOC_TX           0xf021
  60 #define CFS_FAIL_GNI_FMABLK_AVAIL       0xf022
  61 #define CFS_FAIL_GNI_EP_CREATE          0xf023
  62 #define CFS_FAIL_GNI_CQ_GET_EVENT       0xf024
  63 #define CFS_FAIL_GNI_PROBE              0xf025
  64 #define CFS_FAIL_GNI_EP_TEST            0xf026
  65 #define CFS_FAIL_GNI_CONNREQ_DROP       0xf027
  66 #define CFS_FAIL_GNI_CONNREQ_PROTO      0xf028
  67 #define CFS_FAIL_GNI_CONND_PILEUP       0xf029
  68 #define CFS_FAIL_GNI_PHYS_SETUP         0xf02a
  69 #define CFS_FAIL_GNI_FIND_TARGET        0xf02b
  70 #define CFS_FAIL_GNI_WC_DGRAM_FREE      0xf02c
  71 #define CFS_FAIL_GNI_DROP_CLOSING       0xf02d
  72 #define CFS_FAIL_GNI_RX_CLOSE_CLOSING   0xf02e
  73 #define CFS_FAIL_GNI_RX_CLOSE_CLOSED    0xf02f
  74 #define CFS_FAIL_GNI_EP_POST            0xf030
  75 #define CFS_FAIL_GNI_PACK_SRCNID        0xf031
  76 #define CFS_FAIL_GNI_PACK_DSTNID        0xf032
  77 #define CFS_FAIL_GNI_PROBE_WAIT         0xf033
  78 #define CFS_FAIL_GNI_SMSG_CKSUM1        0xf034
  79 #define CFS_FAIL_GNI_SMSG_CKSUM2        0xf035
  80 #define CFS_FAIL_GNI_SMSG_CKSUM3        0xf036
  81 #define CFS_FAIL_GNI_DROP_DESTROY_EP    0xf037
  82 #define CFS_FAIL_GNI_SMSG_GETNEXT       0xf038
  83 #define CFS_FAIL_GNI_FINISH_PURG        0xf039
  84 #define CFS_FAIL_GNI_PURG_REL_DELAY     0xf03a
  85 #define CFS_FAIL_GNI_DONT_NOTIFY        0xf03b
  86 #define CFS_FAIL_GNI_VIRT_SMALL_MAP     0xf03c
  87 #define CFS_FAIL_GNI_DELAY_RDMAQ        0xf03d
  88 #define CFS_FAIL_GNI_PAUSE_SHUTDOWN     0xf03e
  89 #define CFS_FAIL_GNI_PAUSE_DGRAM_COMP   0xf03f
  90 #define CFS_FAIL_GNI_NET_LOOKUP         0xf040
  91 #define CFS_FAIL_GNI_RECV_TIMEOUT       0xf041
  92 #define CFS_FAIL_GNI_SEND_TIMEOUT       0xf042
  93 #define CFS_FAIL_GNI_ONLY_NOOP          0xf043
  94 #define CFS_FAIL_GNI_FINISH_PURG2       0xf044
  95 #define CFS_FAIL_GNI_RACE_RESET         0xf045
  96 #define CFS_FAIL_GNI_GNP_CONNECTING1    0xf046
  97 #define CFS_FAIL_GNI_GNP_CONNECTING2    0xf047
  98 #define CFS_FAIL_GNI_GNP_CONNECTING3    0xf048
  99 #define CFS_FAIL_GNI_PUT_ACK_AGAIN      0xf050
 100 #define CFS_FAIL_GNI_GET_REQ_AGAIN      0xf051
 101
 102 /* helper macros */
 103 extern void
 104 _kgnilnd_api_rc_lbug(const char *rcstr, int rc, struct libcfs_debug_msg_data *data,
 105                         const char *fmt, ...)
 106         __attribute__ ((format (printf, 4, 5)));
 107
 108 #define kgnilnd_api_rc_lbug(msgdata, rc, fmt, a...)                             \
 109 do {                                                                            \
 110         CFS_CHECK_STACK(msgdata, D_ERROR, NULL);                                \
 111         /* we don't mask this - it is always at D_ERROR */                      \
 112         _kgnilnd_api_rc_lbug(kgnilnd_api_rc2str(rc), (rc), msgdata, fmt, ##a);  \
 113 } while (0)
 114
 115 #define DO_RETCODE(x) case x: return #x;
 116 static inline const char *
 117 kgnilnd_api_rc2str(gni_return_t rrc)
 118 {
 119
 120         switch (rrc) {
 121                 DO_RETCODE(GNI_RC_SUCCESS)
 122                 DO_RETCODE(GNI_RC_NOT_DONE);
 123                 DO_RETCODE(GNI_RC_INVALID_PARAM);
 124                 DO_RETCODE(GNI_RC_ERROR_RESOURCE);
 125                 DO_RETCODE(GNI_RC_TIMEOUT);
 126                 DO_RETCODE(GNI_RC_PERMISSION_ERROR);
 127                 DO_RETCODE(GNI_RC_DESCRIPTOR_ERROR);
 128                 DO_RETCODE(GNI_RC_ALIGNMENT_ERROR);
 129                 DO_RETCODE(GNI_RC_INVALID_STATE);
 130                 DO_RETCODE(GNI_RC_NO_MATCH);
 131                 DO_RETCODE(GNI_RC_SIZE_ERROR);
 132                 DO_RETCODE(GNI_RC_TRANSACTION_ERROR);
 133                 DO_RETCODE(GNI_RC_ILLEGAL_OP);
 134                 DO_RETCODE(GNI_RC_ERROR_NOMEM);
 135         }
 136         LBUG();
 137 }
 138 #undef DO_RETCODE
 139
 140 /* log an error and LBUG for unhandled rc from gni api function
 141  * the fmt should be something like:
 142  *  gni_api_call(arg1, arg2, arg3)
 143  */
 144
 145 /* apick_fn and apick_fmt should be defined for each site */
 146 #undef apick_fn
 147 #undef apick_fmt
 148
 149 #define GNILND_API_RC_LBUG(args...)                                             \
 150 do {                                                                            \
 151         LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);                     \
 152         kgnilnd_api_rc_lbug(&msgdata, rrc, apick_fn"("apick_fmt")", ##args);    \
 153 } while (0)
 154
 155 #define GNILND_API_SWBUG(args...)                                               \
 156 do {                                                                            \
 157         CERROR("likely SOFTWARE BUG "apick_fn"("apick_fmt") rc %s\n",           \
 158                  ##args, kgnilnd_api_rc2str(rrc));                              \
 159 } while (0)
 160
 161 #define GNILND_API_EINVAL(args...)                                              \
 162 do {                                                                            \
 163         CERROR("invalid parameter to "apick_fn"("apick_fmt") rc %s\n",          \
 164                  ##args, kgnilnd_api_rc2str(rrc));                              \
 165 } while (0)
 166
 167 #define GNILND_API_RESOURCE(args...)                                            \
 168 do {                                                                            \
 169         CERROR("no resources for "apick_fn"("apick_fmt") rc %s\n",              \
 170                 ##args, kgnilnd_api_rc2str(rrc));                               \
 171 } while (0)
 172
 173 #define GNILND_API_BUSY(args...)                                                \
 174 do {                                                                            \
 175         CERROR("resources busy for "apick_fn"("apick_fmt") rc %s\n",            \
 176                 ##args, kgnilnd_api_rc2str(rrc));                               \
 177 } while (0)
 178
 179 #undef DEBUG_SMSG_CREDITS
 180 #ifdef DEBUG_SMSG_CREDITS
 181 #define CRAY_CONFIG_GHAL_GEMINI
 182 #include <gni_priv.h>
 183 #define GNIDBG_SMSG_CREDS(level, conn)                                        \
 184 do {                                                                          \
 185         gni_ep_smsg_mbox_t *smsg = conn->gnc_ephandle->smsg;                  \
 186         CDEBUG(level, "SMSGDBG: conn %p mcred %d/%d bcred %d/%d "             \
 187                 "s_seq %d/%d/%d r_seq %d/%d/%d retr %d\n",                    \
 188                 conn, smsg->mbox_credits, smsg->back_mbox_credits,            \
 189                 smsg->buffer_credits, smsg->back_buffer_credits,              \
 190                 smsg->s_seqno, smsg->s_seqno_back_mbox_credits,               \
 191                 smsg->s_seqno_back_buffer_credits, smsg->r_seqno,             \
 192                 smsg->r_seqno_back_mbox_credits,                              \
 193                 smsg->r_seqno_back_buffer_credits, smsg->retransmit_count);   \
 194 } while (0)
 195 #else
 196 #define GNIDBG_SMSG_CREDS(level, conn) do {} while(0)
 197 #endif
 198
 199 /* these are all wrappers around gni_XXX functions.
 200  * This allows us to handle all the return codes and api checks without
 201  * dirtying up the logic code */
 202
 203 /* TODO: RETURN wrapper that translates integer to GNI API RC string */
 204
 205 #define apick_fn "kgnilnd_cdm_create"
 206 #define apick_fmt "%u, %u, %u, %u, 0x%p"
 207 static inline gni_return_t kgnilnd_cdm_create(
 208                 IN uint32_t             inst_id,
 209                 IN uint8_t              ptag,
 210                 IN uint32_t             cookie,
 211                 IN uint32_t             modes,
 212                 OUT gni_cdm_handle_t    *cdm_hndl
 213                 )
 214 {
 215         gni_return_t rrc;
 216
 217         /* Error injection */
 218         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CDM_CREATE)) {
 219                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 220         } else {
 221                 rrc = gni_cdm_create(inst_id, ptag, cookie, modes, cdm_hndl);
 222         }
 223
 224         switch (rrc)  {
 225         case GNI_RC_SUCCESS:
 226                 break;
 227         case GNI_RC_ERROR_RESOURCE:
 228         case GNI_RC_INVALID_PARAM:
 229                 /* Try to bail gracefully */
 230                 GNILND_API_SWBUG(
 231                         inst_id, ptag, cookie, modes, cdm_hndl);
 232                 break;
 233         default:
 234                 GNILND_API_RC_LBUG(
 235                         inst_id, ptag, cookie, modes, cdm_hndl);
 236
 237                 /* LBUG never returns, but just for style and consistency */
 238                 break;
 239         }
 240         RETURN(rrc);
 241 }
 242
 243 #undef apick_fn
 244 #undef apick_fmt
 245
 246 #define apick_fn "kgnilnd_cdm_attach"
 247 #define apick_fmt "0x%p, %u, 0x%p, 0x%p"
 248 static inline gni_return_t kgnilnd_cdm_attach(
 249                 IN gni_cdm_handle_t     cdm_hndl,
 250                 IN uint32_t             device_id,
 251                 OUT uint32_t            *local_addr,
 252                 OUT gni_nic_handle_t    *nic_hndl
 253                 )
 254 {
 255         gni_return_t rrc;
 256
 257         /* Error injection */
 258         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CDM_ATTACH)) {
 259                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 260         } else {
 261                 rrc = gni_cdm_attach(cdm_hndl, device_id, local_addr, nic_hndl);
 262         }
 263
 264         switch (rrc)  {
 265         case GNI_RC_SUCCESS:
 266                 break;
 267         case GNI_RC_NO_MATCH:
 268         case GNI_RC_INVALID_PARAM:
 269                 GNILND_API_SWBUG(
 270                         cdm_hndl, device_id, local_addr, nic_hndl);
 271                 break;
 272         case GNI_RC_ERROR_RESOURCE:
 273         case GNI_RC_INVALID_STATE:
 274                 GNILND_API_RESOURCE(
 275                         cdm_hndl, device_id, local_addr, nic_hndl);
 276                 break;
 277         default:
 278                 GNILND_API_RC_LBUG(
 279                         cdm_hndl, device_id, local_addr, nic_hndl);
 280
 281                 /* LBUG never returns, but just for style and consistency */
 282                 break;
 283         }
 284         RETURN(rrc);
 285 }
 286 #undef apick_fmt
 287 #undef apick_fn
 288
 289 #define apick_fn "kgnilnd_cdm_destroy"
 290 #define apick_fmt "0x%p"
 291 static inline gni_return_t kgnilnd_cdm_destroy(
 292                 IN gni_cdm_handle_t     cdm_hndl
 293                 )
 294 {
 295         gni_return_t rrc;
 296
 297         /* Error injection */
 298         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_DESTROY)) {
 299                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 300         } else {
 301                 rrc = gni_cdm_destroy(
 302                         cdm_hndl);
 303         }
 304
 305         switch (rrc)  {
 306         case GNI_RC_SUCCESS:
 307                 break;
 308         case GNI_RC_INVALID_PARAM:
 309                 GNILND_API_SWBUG(
 310                         cdm_hndl);
 311                 break;
 312         default:
 313                 GNILND_API_RC_LBUG(
 314                         cdm_hndl);
 315
 316                 /* LBUG never returns, but just for style and consistency */
 317                 break;
 318         }
 319         RETURN(rrc);
 320 }
 321 #undef apick_fn
 322 #undef apick_fmt
 323
 324 #define apick_fn "kgnilnd_subscribe_errors"
 325 #define apick_fmt "0x%p,%x,%u,0x%p,0x%p,0x%p"
 326 static inline gni_return_t kgnilnd_subscribe_errors(
 327                 IN gni_nic_handle_t  nic_handle,
 328                 IN gni_error_mask_t  mask,
 329                 IN uint32_t          EEQ_size,
 330                 IN void              (*EQ_new_event)(gni_err_handle_t),
 331                 IN void              (*app_crit_err)(gni_err_handle_t),
 332                 OUT gni_err_handle_t *err_handle
 333                 )
 334 {
 335         gni_return_t rrc;
 336
 337         /* Error injection */
 338         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_ERR_SUBSCRIBE)) {
 339                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 340         } else {
 341                 rrc = gni_subscribe_errors(
 342                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 343                         err_handle);
 344         }
 345
 346         switch (rrc)  {
 347         case GNI_RC_SUCCESS:
 348                 break;
 349         case GNI_RC_INVALID_PARAM:
 350                 GNILND_API_SWBUG(
 351                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 352                         err_handle);
 353                 break;
 354         case GNI_RC_ERROR_RESOURCE:
 355                 GNILND_API_RESOURCE(
 356                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 357                         err_handle);
 358                 break;
 359         default:
 360                 GNILND_API_RC_LBUG(
 361                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 362                         err_handle);
 363                 /* LBUG never returns, but just for style and consistency */
 364                 break;
 365         }
 366         RETURN(rrc);
 367 }
 368 #undef apick_fn
 369 #undef apick_fmt
 370
 371 #define apick_fn "kgnilnd_release_errors"
 372 #define apick_fmt "0x%p"
 373 static inline gni_return_t kgnilnd_release_errors(
 374                 IN gni_err_handle_t err_handle
 375                 )
 376 {
 377         gni_return_t rrc;
 378
 379         rrc = gni_release_errors(
 380                         err_handle);
 381
 382         switch (rrc)  {
 383         case GNI_RC_SUCCESS:
 384                 break;
 385         case GNI_RC_INVALID_PARAM:
 386         case GNI_RC_NOT_DONE:
 387                 GNILND_API_SWBUG(
 388                         err_handle);
 389                 break;
 390         default:
 391                 GNILND_API_RC_LBUG(
 392                         err_handle);
 393                 /* LBUG never returns, but just for style and consistency */
 394                 break;
 395         }
 396         RETURN(rrc);
 397 }
 398 #undef apick_fn
 399 #undef apick_fmt
 400
 401 #define apick_fn "kgnilnd_set_quiesce_callback"
 402 #define apick_fmt "0x%p,0x%p"
 403 static inline gni_return_t kgnilnd_set_quiesce_callback(
 404                 IN gni_nic_handle_t  nic_handle,
 405                 IN void              (*qsce_func)(gni_nic_handle_t, uint64_t msecs)
 406                 )
 407 {
 408         gni_return_t rrc;
 409
 410         /* Error injection */
 411         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_REG_QUIESCE)) {
 412                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 413         } else {
 414                 rrc = gni_set_quiesce_callback(
 415                         nic_handle, qsce_func);
 416         }
 417
 418         switch (rrc)  {
 419         case GNI_RC_SUCCESS:
 420                 break;
 421         case GNI_RC_INVALID_STATE:
 422         case GNI_RC_INVALID_PARAM:
 423                 GNILND_API_SWBUG(
 424                         nic_handle, qsce_func);
 425                 break;
 426         default:
 427                 GNILND_API_RC_LBUG(
 428                         nic_handle, qsce_func);
 429                 /* LBUG never returns, but just for style and consistency */
 430                 break;
 431         }
 432         RETURN(rrc);
 433 }
 434 #undef apick_fn
 435 #undef apick_fmt
 436
 437 #define apick_fn "kgnilnd_get_quiesce_status"
 438 #define apick_fmt "0x%p"
 439 static inline gni_return_t kgnilnd_get_quiesce_status(
 440                 IN gni_nic_handle_t  nic_handle
 441                 )
 442 {
 443         uint32_t rrc;
 444
 445         /* this has weird RC -
 446          * 0 - quiesce not in progress
 447          * 1 - quiesce is turned on
 448         */
 449
 450         /* Error injection */
 451         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_IN_QUIESCE)) {
 452                 rrc = 1;
 453         } else {
 454                 rrc = gni_get_quiesce_status(
 455                         nic_handle);
 456         }
 457
 458         switch (rrc)  {
 459         case 1:
 460         case 0:
 461                 break;
 462         default:
 463                 GNILND_API_RC_LBUG(
 464                         nic_handle);
 465                 /* LBUG never returns, but just for style and consistency */
 466                 break;
 467         }
 468         RETURN(rrc);
 469 }
 470 #undef apick_fn
 471 #undef apick_fmt
 472
 473 #define apick_fn "kgnilnd_cq_create"
 474 #define apick_fmt "0x%p, %u, %u, 0x%p, "LPX64", 0x%p"
 475 static inline gni_return_t kgnilnd_cq_create(
 476                 IN gni_nic_handle_t     nic_hndl,
 477                 IN uint32_t             entry_count,
 478                 IN uint32_t             delay_index,
 479                 IN gni_cq_event_hndlr_f *event_handler,
 480                 IN uint64_t             usr_event_data,
 481                 OUT gni_cq_handle_t     *cq_hndl
 482                 )
 483 {
 484         gni_return_t rrc;
 485
 486         /* Error injection */
 487         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_CREATE)) {
 488                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 489         } else {
 490                 rrc = gni_cq_create(
 491                        nic_hndl, entry_count, delay_index, event_handler,
 492                         usr_event_data, cq_hndl);
 493         }
 494
 495         switch (rrc)  {
 496         case GNI_RC_SUCCESS:
 497                 break;
 498         case GNI_RC_INVALID_PARAM:
 499                 GNILND_API_SWBUG(
 500                         nic_hndl, entry_count, delay_index, event_handler,
 501                         usr_event_data, cq_hndl);
 502                 break;
 503         case GNI_RC_ERROR_RESOURCE:
 504                 GNILND_API_RESOURCE(
 505                         nic_hndl, entry_count, delay_index, event_handler,
 506                         usr_event_data, cq_hndl);
 507                 break;
 508         default:
 509                 GNILND_API_RC_LBUG(
 510                         nic_hndl, entry_count, delay_index, event_handler,
 511                         usr_event_data, cq_hndl);
 512
 513                 /* LBUG never returns, but just for style and consistency */
 514                 break;
 515         }
 516         RETURN(rrc);
 517 }
 518 #undef apick_fn
 519 #undef apick_fmt
 520
 521 #define apick_fn "kgnilnd_cq_destroy"
 522 #define apick_fmt "0x%p"
 523 static inline gni_return_t kgnilnd_cq_destroy(
 524                 IN gni_cq_handle_t cq_hndl
 525                 )
 526 {
 527         gni_return_t rrc;
 528
 529         /* Error injection */
 530         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_DESTROY)) {
 531                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 532         } else {
 533
 534                 rrc = gni_cq_destroy(
 535                         cq_hndl);
 536         }
 537
 538         switch (rrc)  {
 539         case GNI_RC_SUCCESS:
 540                 break;
 541         case GNI_RC_INVALID_PARAM:
 542                 GNILND_API_SWBUG(
 543                         cq_hndl);
 544                 break;
 545         case GNI_RC_ERROR_RESOURCE:
 546                 GNILND_API_BUSY(
 547                         cq_hndl);
 548                 break;
 549         default:
 550                 GNILND_API_RC_LBUG(
 551                         cq_hndl);
 552
 553                 /* LBUG never returns, but just for style and consistency */
 554                 break;
 555         }
 556         RETURN(rrc);
 557 }
 558 #undef apick_fn
 559 #undef apick_fmt
 560
 561 #define apick_fn "kgnilnd_cq_get_event"
 562 #define apick_fmt "0x%p, 0x%p"
 563 static inline gni_return_t kgnilnd_cq_get_event(
 564                 IN gni_cq_handle_t cq_hndl,
 565                 OUT gni_cq_entry_t *event_data
 566                 )
 567 {
 568         gni_return_t rrc;
 569
 570         /* no error injection - CQs are touchy about the data.
 571          * where appropriate, we'll do this on the CQs that should be able to
 572          * handle the various errors */
 573         rrc = gni_cq_get_event(
 574                         cq_hndl, event_data);
 575
 576         switch (rrc)  {
 577         case GNI_RC_SUCCESS:
 578         case GNI_RC_NOT_DONE:
 579         case GNI_RC_TRANSACTION_ERROR:
 580                 break;
 581         case GNI_RC_ERROR_RESOURCE:
 582                 LASSERTF(GNI_CQ_OVERRUN(*event_data),
 583                          "kgni returned ERROR_RESOURCE but cq_hndl 0x%p is not "
 584                          "overrun\n", cq_hndl);
 585                 break;
 586         case GNI_RC_INVALID_PARAM:
 587                 GNILND_API_SWBUG(
 588                         cq_hndl, event_data);
 589                 break;
 590         default:
 591                 GNILND_API_RC_LBUG(
 592                         cq_hndl, event_data);
 593
 594                 /* LBUG never returns, but just for style and consistency */
 595                 break;
 596         }
 597         return rrc;
 598 }
 599 #undef apick_fn
 600 #undef apick_fmt
 601
 602 #define apick_fn "kgnilnd_smsg_init"
 603 #define apick_fmt "0x%p, 0x%p, 0x%p"
 604 static inline gni_return_t kgnilnd_smsg_init(
 605                 IN gni_ep_handle_t      ep_hndl,
 606                 IN gni_smsg_attr_t      *local_smsg_attr,
 607                 IN gni_smsg_attr_t      *remote_smsg_attr
 608                 )
 609 {
 610         gni_return_t rrc;
 611
 612         /* Error injection */
 613         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_INIT)) {
 614                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_RESOURCE;
 615         } else {
 616                 rrc = gni_smsg_init(
 617                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 618         }
 619
 620         switch (rrc)  {
 621         /* both of these are OK, upper SW needs to handle */
 622         case GNI_RC_SUCCESS:
 623         case GNI_RC_NOT_DONE:
 624                 break;
 625         case GNI_RC_INVALID_PARAM:
 626         case GNI_RC_INVALID_STATE:
 627                 GNILND_API_SWBUG(
 628                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 629                 break;
 630         case GNI_RC_ERROR_RESOURCE:
 631                 GNILND_API_RESOURCE(
 632                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 633                 break;
 634         default:
 635                 GNILND_API_RC_LBUG(
 636                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 637
 638                 /* LBUG never returns, but just for style and consistency */
 639                 break;
 640         }
 641         RETURN(rrc);
 642 }
 643 #undef apick_fn
 644 #undef apick_fmt
 645
 646 #define apick_fn "kgnilnd_smsg_send"
 647 #define apick_fmt "0x%p, 0x%p, %d, 0x%p, %u %u"
 648 static inline gni_return_t kgnilnd_smsg_send(
 649                 IN gni_ep_handle_t      ep_hndl,
 650                 IN void                 *header,
 651                 IN uint32_t             header_length,
 652                 IN void                 *data,
 653                 IN uint32_t             data_length,
 654                 IN uint32_t             msg_id
 655                 )
 656 {
 657         gni_return_t rrc;
 658
 659         /* Error injection */
 660         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_SEND)) {
 661                 if (cfs_fail_loc & CFS_FAIL_RAND) {
 662                         rrc = GNI_RC_NOT_DONE;
 663                 } else {
 664                         rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 665                 }
 666         } else {
 667                 rrc = gni_smsg_send(
 668                         ep_hndl, header, header_length, data, data_length, msg_id);
 669         }
 670
 671         switch (rrc)  {
 672         /* both of these are OK, upper SW needs to handle */
 673         case GNI_RC_SUCCESS:
 674         case GNI_RC_NOT_DONE:
 675                 break;
 676         case GNI_RC_INVALID_PARAM:
 677                 GNILND_API_SWBUG(
 678                         ep_hndl, header, header_length, data, data_length, msg_id);
 679                 break;
 680         case GNI_RC_ERROR_RESOURCE:
 681                 GNILND_API_RESOURCE(
 682                         ep_hndl, header, header_length, data, data_length, msg_id);
 683                 break;
 684         default:
 685                 GNILND_API_RC_LBUG(
 686                         ep_hndl, header, header_length, data, data_length, msg_id);
 687
 688                 /* LBUG never returns, but just for style and consistency */
 689                 break;
 690         }
 691         RETURN(rrc);
 692 }
 693 #undef apick_fn
 694 #undef apick_fmt
 695
 696 #define apick_fn "kgnilnd_smsg_getnext"
 697 #define apick_fmt "0x%p,0x%p"
 698 static inline gni_return_t kgnilnd_smsg_getnext(
 699                 IN gni_ep_handle_t      ep_hndl,
 700                 OUT void                **header
 701                 )
 702 {
 703         gni_return_t rrc;
 704
 705         /* Error injection */
 706         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_RELEASE)) {
 707                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 708         } else {
 709                 rrc = gni_smsg_getnext(
 710                         ep_hndl, header);
 711         }
 712
 713         switch (rrc)  {
 714         /* both of these are OK, upper SW needs to handle */
 715         case GNI_RC_SUCCESS:
 716         case GNI_RC_NOT_DONE:
 717         case GNI_RC_INVALID_STATE:
 718                 break;
 719         case GNI_RC_INVALID_PARAM:
 720                 GNILND_API_SWBUG(
 721                         ep_hndl, header);
 722                 break;
 723         default:
 724                 GNILND_API_RC_LBUG(
 725                         ep_hndl, header);
 726
 727                 /* LBUG never returns, but just for style and consistency */
 728                 break;
 729         }
 730         RETURN(rrc);
 731 }
 732 #undef apick_fn
 733 #undef apick_fmt
 734
 735 #define apick_fn "kgnilnd_smsg_release"
 736 #define apick_fmt "0x%p"
 737 static inline gni_return_t kgnilnd_smsg_release(
 738                 IN gni_ep_handle_t      ep_hndl
 739                 )
 740 {
 741         gni_return_t rrc;
 742
 743         /* Error injection */
 744         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_RELEASE)) {
 745                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 746         } else {
 747                 rrc = gni_smsg_release(
 748                         ep_hndl);
 749         }
 750
 751         switch (rrc)  {
 752         /* both of these are OK, upper SW needs to handle */
 753         case GNI_RC_SUCCESS:
 754         case GNI_RC_NOT_DONE:
 755                 break;
 756         case GNI_RC_INVALID_PARAM:
 757                 GNILND_API_SWBUG(
 758                         ep_hndl);
 759                 break;
 760         default:
 761                 GNILND_API_RC_LBUG(
 762                         ep_hndl);
 763
 764                 /* LBUG never returns, but just for style and consistency */
 765                 break;
 766         }
 767         RETURN(rrc);
 768 }
 769 #undef apick_fn
 770 #undef apick_fmt
 771
 772 #define apick_fn "kgnilnd_ep_create"
 773 #define apick_fmt "0x%p, 0x%p, 0x%p"
 774 static inline gni_return_t kgnilnd_ep_create(
 775                 IN gni_nic_handle_t     nic_hndl,
 776                 IN gni_cq_handle_t      src_cq_hndl,
 777                 OUT gni_ep_handle_t     *ep_hndl
 778                 )
 779 {
 780         gni_return_t rrc;
 781
 782         /* error injection */
 783         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_CREATE)) {
 784                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_NOMEM;
 785         } else {
 786                 rrc = gni_ep_create(
 787                         nic_hndl, src_cq_hndl, ep_hndl);
 788         }
 789
 790         switch (rrc)  {
 791         case GNI_RC_SUCCESS:
 792                 break;
 793         case GNI_RC_INVALID_PARAM:
 794                 GNILND_API_SWBUG(
 795                         nic_hndl, src_cq_hndl, ep_hndl);
 796                 break;
 797         case GNI_RC_ERROR_NOMEM:
 798                 GNILND_API_RESOURCE(
 799                         nic_hndl, src_cq_hndl, ep_hndl);
 800                 break;
 801         default:
 802                 GNILND_API_RC_LBUG(
 803                         nic_hndl, src_cq_hndl, ep_hndl);
 804
 805                 /* lbug never returns, but just for style and consistency */
 806                 break;
 807         }
 808         RETURN(rrc);
 809 }
 810 #undef apick_fn
 811 #undef apick_fmt
 812
 813 #define apick_fn "kgnilnd_ep_bind"
 814 #define apick_fmt "0x%p, %x, %x"
 815 static inline gni_return_t kgnilnd_ep_bind(
 816                 IN gni_ep_handle_t      ep_hndl,
 817                 IN uint32_t             remote_addr,
 818                 IN uint32_t             remote_id
 819                 )
 820 {
 821         gni_return_t rrc;
 822
 823         /* error injection */
 824         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_BIND)) {
 825                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 826         } else {
 827                 rrc = gni_ep_bind(
 828                         ep_hndl, remote_addr, remote_id);
 829         }
 830
 831         switch (rrc)  {
 832         /* both of these are ok, upper sw needs to handle */
 833         case GNI_RC_SUCCESS:
 834         case GNI_RC_NOT_DONE:
 835                 break;
 836         case GNI_RC_INVALID_PARAM:
 837                 GNILND_API_SWBUG(
 838                         ep_hndl, remote_addr, remote_id);
 839                 break;
 840         default:
 841                 GNILND_API_RC_LBUG(
 842                         ep_hndl, remote_addr, remote_id);
 843
 844                 /* lbug never returns, but just for style and consistency */
 845                 break;
 846         }
 847         RETURN(rrc);
 848 }
 849 #undef apick_fn
 850 #undef apick_fmt
 851
 852 #define apick_fn "kgnilnd_ep_set_eventdata"
 853 #define apick_fmt "0x%p, %x, %x"
 854 static inline gni_return_t kgnilnd_ep_set_eventdata(
 855                 IN gni_ep_handle_t      ep_hndl,
 856                 IN uint32_t             local_event,
 857                 IN uint32_t             remote_event
 858                 )
 859 {
 860         gni_return_t rrc;
 861
 862         /* Error injection */
 863         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_SET_EVDATA)) {
 864                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 865         } else {
 866                 rrc = gni_ep_set_eventdata(
 867                         ep_hndl, local_event, remote_event);
 868         }
 869
 870         switch (rrc)  {
 871         case GNI_RC_SUCCESS:
 872                 break;
 873         case GNI_RC_INVALID_PARAM:
 874                 GNILND_API_SWBUG(
 875                         ep_hndl, local_event, remote_event);
 876                 break;
 877         default:
 878                 GNILND_API_RC_LBUG(
 879                         ep_hndl, local_event, remote_event);
 880
 881                 /* LBUG never returns, but just for style and consistency */
 882                 break;
 883         }
 884         RETURN(rrc);
 885 }
 886 #undef apick_fn
 887 #undef apick_fmt
 888
 889 #define apick_fn "kgnilnd_ep_unbind"
 890 #define apick_fmt "0x%p"
 891 static inline gni_return_t kgnilnd_ep_unbind(
 892                 IN gni_ep_handle_t      ep_hndl
 893                 )
 894 {
 895         gni_return_t rrc;
 896
 897         /* Error injection */
 898         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_UNBIND)) {
 899                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 900         } else {
 901                 rrc = gni_ep_unbind(
 902                         ep_hndl);
 903         }
 904
 905         switch (rrc)  {
 906         /* both of these are OK, upper SW needs to handle */
 907         case GNI_RC_NOT_DONE:
 908         case GNI_RC_SUCCESS:
 909                 break;
 910         case GNI_RC_INVALID_PARAM:
 911                 GNILND_API_SWBUG(
 912                         ep_hndl);
 913                 break;
 914         default:
 915                 GNILND_API_RC_LBUG(
 916                         ep_hndl);
 917
 918                 /* LBUG never returns, but just for style and consistency */
 919                 break;
 920         }
 921         RETURN(rrc);
 922 }
 923 #undef apick_fn
 924 #undef apick_fmt
 925
 926 #define apick_fn "kgnilnd_ep_destroy"
 927 #define apick_fmt "0x%p"
 928 static inline gni_return_t kgnilnd_ep_destroy(
 929                 IN gni_ep_handle_t      ep_hndl
 930                 )
 931 {
 932         gni_return_t rrc;
 933
 934         /* Error injection */
 935         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_DESTROY)) {
 936                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 937         } else {
 938                 rrc = gni_ep_destroy(
 939                         ep_hndl);
 940         }
 941
 942         switch (rrc)  {
 943         case GNI_RC_SUCCESS:
 944                 break;
 945         case GNI_RC_INVALID_PARAM:
 946                 GNILND_API_SWBUG(
 947                         ep_hndl);
 948                 break;
 949         default:
 950                 GNILND_API_RC_LBUG(
 951                         ep_hndl);
 952
 953                 /* LBUG never returns, but just for style and consistency */
 954                 break;
 955         }
 956         RETURN(rrc);
 957 }
 958 #undef apick_fn
 959 #undef apick_fmt
 960
 961 #define apick_fn "kgnilnd_ep_postdata_w_id"
 962 #define apick_fmt "0x%p, 0x%p, %d, 0x%p, %d, "LPU64""
 963 static inline gni_return_t kgnilnd_ep_postdata_w_id(
 964                 IN gni_ep_handle_t ep_hndl,
 965                 IN void            *in_data,
 966                 IN uint16_t        data_len,
 967                 IN void            *out_buf,
 968                 IN uint16_t        buf_size,
 969                 IN uint64_t        datagram_id
 970                 )
 971 {
 972         gni_return_t rrc;
 973
 974         /* Error injection */
 975         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_POST)) {
 976                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_SIZE_ERROR;
 977         } else {
 978                 rrc = gni_ep_postdata_w_id(
 979                         ep_hndl, in_data, data_len, out_buf, buf_size,
 980                         datagram_id);
 981         }
 982
 983         switch (rrc)  {
 984         case GNI_RC_SUCCESS:
 985         case GNI_RC_ERROR_NOMEM:
 986         case GNI_RC_ERROR_RESOURCE:
 987                 break;
 988         case GNI_RC_INVALID_PARAM:
 989         case GNI_RC_SIZE_ERROR:
 990                 GNILND_API_SWBUG(
 991                         ep_hndl, in_data, data_len, out_buf, buf_size,
 992                         datagram_id);
 993                 break;
 994         default:
 995                 GNILND_API_RC_LBUG(
 996                         ep_hndl, in_data, data_len, out_buf, buf_size,
 997                         datagram_id);
 998
 999                 /* LBUG never returns, but just for style and consistency */
1000                 break;
1001         }
1002         RETURN(rrc);
1003 }
1004 #undef apick_fn
1005 #undef apick_fmt
1006
1007 #define apick_fn "kgnilnd_ep_postdata_test_by_id"
1008 #define apick_fmt "0x%p, "LPU64", 0x%p, 0x%p, 0x%p"
1009 static inline gni_return_t kgnilnd_ep_postdata_test_by_id(
1010                 IN gni_ep_handle_t      ep_hndl,
1011                 IN uint64_t             datagram_id,
1012                 OUT gni_post_state_t    *post_state,
1013                 OUT uint32_t            *remote_addr,
1014                 OUT uint32_t            *remote_id
1015                 )
1016 {
1017         gni_return_t rrc;
1018
1019         /* Error injection */
1020         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_TEST)) {
1021                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_NOMEM;
1022         } else {
1023                 rrc = gni_ep_postdata_test_by_id(
1024                         ep_hndl, datagram_id, post_state, remote_addr,
1025                         remote_id);
1026
1027                 /* we want to lie, but we need to do the actual work first
1028                  * so we don't keep getting the event saying a dgram is ready */
1029                 if (rrc == GNI_RC_SUCCESS && CFS_FAIL_CHECK(CFS_FAIL_GNI_DG_TERMINATE)) {
1030                         /* don't use fail_val, allows us to do FAIL_SOME */
1031                         *post_state = GNI_POST_TERMINATED;
1032                 }
1033         }
1034
1035         switch (rrc)  {
1036         case GNI_RC_SUCCESS:
1037         case GNI_RC_NO_MATCH:
1038                 break;
1039         case GNI_RC_SIZE_ERROR:
1040         case GNI_RC_INVALID_PARAM:
1041                 GNILND_API_SWBUG(
1042                         ep_hndl, datagram_id, post_state, remote_addr,
1043                         remote_id);
1044                 break;
1045         case GNI_RC_ERROR_NOMEM:
1046                 GNILND_API_RESOURCE(
1047                         ep_hndl, datagram_id, post_state, remote_addr,
1048                         remote_id);
1049                 break;
1050         default:
1051                 GNILND_API_RC_LBUG(
1052                         ep_hndl, datagram_id, post_state, remote_addr,
1053                         remote_id);
1054
1055                 /* LBUG never returns, but just for style and consistency */
1056                 break;
1057         }
1058         RETURN(rrc);
1059 }
1060 #undef apick_fn
1061 #undef apick_fmt
1062
1063 #define apick_fn "kgnilnd_ep_postdata_cancel_by_id"
1064 #define apick_fmt "0x%p, "LPU64""
1065 static inline gni_return_t kgnilnd_ep_postdata_cancel_by_id(
1066                 IN gni_ep_handle_t      ep_hndl,
1067                 IN uint64_t             datagram_id
1068                 )
1069 {
1070         gni_return_t rrc;
1071
1072         /* no error injection as the only thing we'd do is LBUG */
1073
1074         rrc = gni_ep_postdata_cancel_by_id(
1075                 ep_hndl, datagram_id);
1076
1077         switch (rrc)  {
1078         case GNI_RC_SUCCESS:
1079         case GNI_RC_NO_MATCH:
1080                 break;
1081         case GNI_RC_INVALID_PARAM:
1082                 GNILND_API_SWBUG(
1083                         ep_hndl, datagram_id);
1084                 break;
1085         default:
1086                 GNILND_API_RC_LBUG(
1087                         ep_hndl, datagram_id);
1088
1089                 /* LBUG never returns, but just for style and consistency */
1090                 break;
1091         }
1092         RETURN(rrc);
1093 }
1094 #undef apick_fn
1095 #undef apick_fmt
1096
1097 #define apick_fn "kgnilnd_postdata_probe_by_id"
1098 #define apick_fmt "0x%p, 0x%p"
1099 static inline gni_return_t kgnilnd_postdata_probe_by_id(
1100                 IN gni_nic_handle_t    nic_hndl,
1101                 OUT uint64_t          *datagram_id
1102                 )
1103 {
1104         gni_return_t rrc;
1105
1106         /* Error injection */
1107         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PROBE)) {
1108                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NO_MATCH;
1109         } else {
1110                 rrc = gni_postdata_probe_by_id(
1111                         nic_hndl, datagram_id);
1112         }
1113
1114         switch (rrc)  {
1115         case GNI_RC_SUCCESS:
1116         case GNI_RC_NO_MATCH:
1117                 break;
1118         case GNI_RC_INVALID_PARAM:
1119                 GNILND_API_SWBUG(
1120                         nic_hndl, datagram_id);
1121                 break;
1122         default:
1123                 GNILND_API_RC_LBUG(
1124                         nic_hndl, datagram_id);
1125
1126                 /* LBUG never returns, but just for style and consistency */
1127                 break;
1128         }
1129         RETURN(rrc);
1130 }
1131 #undef apick_fn
1132 #undef apick_fmt
1133
1134 #define apick_fn "kgnilnd_postdata_probe_wait_by_id"
1135 #define apick_fmt "0x%p, %d, 0x%p"
1136 static inline gni_return_t kgnilnd_postdata_probe_wait_by_id(
1137                 IN gni_nic_handle_t nic_hndl,
1138                 IN uint32_t         timeout,
1139                 OUT uint64_t        *datagram_id
1140                 )
1141 {
1142         gni_return_t rrc;
1143
1144         /* Error injection */
1145         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PROBE_WAIT)) {
1146                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_TIMEOUT;
1147         } else {
1148                 rrc = gni_postdata_probe_wait_by_id(
1149                         nic_hndl, timeout, datagram_id);
1150         }
1151
1152         switch (rrc)  {
1153         case GNI_RC_SUCCESS:
1154         case GNI_RC_TIMEOUT:
1155                 break;
1156         case GNI_RC_INVALID_PARAM:
1157                 GNILND_API_SWBUG(
1158                         nic_hndl, timeout, datagram_id);
1159                 break;
1160         default:
1161                 GNILND_API_RC_LBUG(
1162                         nic_hndl, timeout, datagram_id);
1163
1164                 /* LBUG never returns, but just for style and consistency */
1165                 break;
1166         }
1167         RETURN(rrc);
1168 }
1169 #undef apick_fn
1170 #undef apick_fmt
1171
1172 #define apick_fn "kgnilnd_post_rdma"
1173 #define apick_fmt "0x%p, 0x%p"
1174 static inline gni_return_t kgnilnd_post_rdma(
1175                 IN gni_ep_handle_t               ep_hndl,
1176                 IN gni_post_descriptor_t        *post_descr
1177                 )
1178 {
1179         gni_return_t rrc;
1180
1181         /* Error injection */
1182         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_POST_RDMA)) {
1183                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
1184         } else {
1185                 rrc = gni_post_rdma(
1186                         ep_hndl, post_descr);
1187         }
1188
1189         switch (rrc)  {
1190         case GNI_RC_SUCCESS:
1191                 break;
1192         case GNI_RC_ALIGNMENT_ERROR:
1193         case GNI_RC_INVALID_PARAM:
1194                 GNILND_API_SWBUG(
1195                         ep_hndl, post_descr);
1196                 break;
1197         case GNI_RC_ERROR_RESOURCE:
1198                 GNILND_API_RESOURCE(
1199                         ep_hndl, post_descr);
1200                 break;
1201         default:
1202                 GNILND_API_RC_LBUG(
1203                         ep_hndl, post_descr);
1204
1205                 /* LBUG never returns, but just for style and consistency */
1206                 break;
1207         }
1208         RETURN(rrc);
1209 }
1210 #undef apick_fn
1211 #undef apick_fmt
1212
1213 #define apick_fn "kgnilnd_get_completed"
1214 #define apick_fmt "0x%p,"LPX64",0x%p"
1215 static inline gni_return_t kgnilnd_get_completed(
1216                 IN gni_cq_handle_t              cq_hndl,
1217                 IN gni_cq_entry_t               event_data,
1218                 OUT gni_post_descriptor_t       **post_descr
1219                 )
1220 {
1221         gni_return_t rrc;
1222
1223
1224         rrc = gni_get_completed(cq_hndl, event_data, post_descr);
1225
1226         switch (rrc)  {
1227         case GNI_RC_TRANSACTION_ERROR:
1228         case GNI_RC_SUCCESS:
1229                 break;
1230         case GNI_RC_DESCRIPTOR_ERROR:
1231         case GNI_RC_INVALID_PARAM:
1232                 GNILND_API_SWBUG(cq_hndl, event_data, post_descr);
1233                 break;
1234         default:
1235                 GNILND_API_RC_LBUG(cq_hndl, event_data, post_descr);
1236                 /* LBUG never returns, but just for style and consistency */
1237                 break;
1238         }
1239
1240         /* Error injection - we need a valid desc, so let kgni give us one
1241          * - then we lie  */
1242         if (rrc == GNI_RC_SUCCESS &&
1243             (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED))) {
1244                 /* We only trigger TRANSACTION_ERROR for now */
1245                 gni_post_descriptor_t *desc;
1246                 rrc = GNI_RC_TRANSACTION_ERROR;
1247                 desc = *post_descr;
1248                 desc->status = rrc;
1249                 /* recoverable decision made from cfs_fail_val in
1250                  *  kgnilnd_cq_error_str and
1251                  *  kgnilnd_cq_error_recoverable */
1252         }
1253         RETURN(rrc);
1254 }
1255 #undef apick_fn
1256 #undef apick_fmt
1257
1258 #define apick_fn "kgnilnd_cq_error_str"
1259 #define apick_fmt LPX64",0x%p,%d"
1260 static inline gni_return_t kgnilnd_cq_error_str(
1261                 IN gni_cq_entry_t       entry,
1262                 IN void                *buffer,
1263                 IN uint32_t             len
1264                 )
1265 {
1266         gni_return_t rrc;
1267
1268         /* Error injection - set string if we injected a
1269          *  TRANSACTION_ERROR earlier */
1270         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED)) {
1271                 /* if we just set persistent error, we can't ever
1272                  * break in via ssh to clear, so use a count > 10 to indicate fatal */
1273                 sprintf(buffer, "INJECT:%s", cfs_fail_val > 10 ?
1274                         "FATAL" : "RECOVERABLE");
1275                 rrc = GNI_RC_SUCCESS;
1276         } else {
1277                 rrc = gni_cq_error_str(
1278                         entry, buffer, len);
1279         }
1280
1281         switch (rrc)  {
1282         case GNI_RC_SUCCESS:
1283                 break;
1284         case GNI_RC_SIZE_ERROR:
1285         case GNI_RC_INVALID_PARAM:
1286                 GNILND_API_SWBUG(
1287                         entry, buffer, len);
1288                 /* give them something to use */
1289                 snprintf(buffer, len, "UNDEF:UNDEF");
1290                 break;
1291         default:
1292                 GNILND_API_RC_LBUG(
1293                         entry, buffer, len);
1294
1295                 /* LBUG never returns, but just for style and consistency */
1296                 break;
1297         }
1298         RETURN(rrc);
1299 }
1300 #undef apick_fn
1301 #undef apick_fmt
1302
1303 #define apick_fn "kgnilnd_cq_error_recoverable"
1304 #define apick_fmt LPX64",0x%p"
1305 static inline gni_return_t kgnilnd_cq_error_recoverable(
1306                 IN gni_cq_entry_t       entry,
1307                 IN uint32_t            *recoverable
1308                 )
1309 {
1310         gni_return_t rrc;
1311
1312         /* Error injection - set string if we injected a
1313          *  TRANSACTION_ERROR earlier */
1314         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED)) {
1315                 *recoverable = cfs_fail_val > 10 ? 0 : 1;
1316                 rrc = GNI_RC_SUCCESS;
1317         } else {
1318                 rrc = gni_cq_error_recoverable(
1319                         entry, recoverable);
1320         }
1321
1322         switch (rrc)  {
1323         case GNI_RC_SUCCESS:
1324                 break;
1325         case GNI_RC_INVALID_STATE:
1326         case GNI_RC_INVALID_PARAM:
1327                 GNILND_API_SWBUG(
1328                         entry, recoverable);
1329                 *recoverable = 0;
1330                 break;
1331         default:
1332                 GNILND_API_RC_LBUG(
1333                         entry, recoverable);
1334
1335                 /* LBUG never returns, but just for style and consistency */
1336                 break;
1337         }
1338         RETURN(rrc);
1339 }
1340 #undef apick_fn
1341 #undef apick_fmt
1342
1343 #define apick_fn "kgnilnd_mem_register_segments"
1344 #define apick_fmt "0x%p,0x%p,%u,0x%p,%x,0x%p"
1345 static inline gni_return_t
1346 kgnilnd_mem_register_segments(
1347                 IN gni_nic_handle_t     nic_hndl,
1348                 IN gni_mem_segment_t    *mem_segments,
1349                 IN uint32_t             segments_cnt,
1350                 IN gni_cq_handle_t      dst_cq_hndl,
1351                 IN uint32_t             flags,
1352                 OUT gni_mem_handle_t    *mem_hndl
1353                 )
1354 {
1355         gni_return_t rrc;
1356
1357         /* Error injection */
1358         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PHYS_MAP)) {
1359                 rrc = GNI_RC_ERROR_RESOURCE;
1360         } else {
1361                 rrc = gni_mem_register_segments(
1362                         nic_hndl, mem_segments, segments_cnt,
1363                         dst_cq_hndl, flags, mem_hndl);
1364         }
1365
1366         switch (rrc)  {
1367         case GNI_RC_SUCCESS:
1368         case GNI_RC_ERROR_RESOURCE:
1369                 break;
1370         case GNI_RC_INVALID_PARAM:
1371                 GNILND_API_SWBUG(
1372                         nic_hndl, mem_segments, segments_cnt,
1373                         dst_cq_hndl, flags, mem_hndl);
1374                 break;
1375         default:
1376                 GNILND_API_RC_LBUG(
1377                         nic_hndl, mem_segments, segments_cnt,
1378                         dst_cq_hndl, flags, mem_hndl);
1379
1380                 /* LBUG never returns, but just for style and consistency */
1381                 break;
1382         }
1383         RETURN(rrc);
1384 }
1385 #undef apick_fn
1386 #undef apick_fmt
1387
1388 #define apick_fn "kgnilnd_mem_register"
1389 #define apick_fmt "0x%p,"LPX64","LPX64"0x%p,%u,0x%p"
1390 static inline gni_return_t kgnilnd_mem_register(
1391                 IN gni_nic_handle_t     nic_hndl,
1392                 IN uint64_t             address,
1393                 IN uint64_t             length,
1394                 IN gni_cq_handle_t      dst_cq_hndl,
1395                 IN uint32_t             flags,
1396                 OUT gni_mem_handle_t    *mem_hndl
1397                 )
1398 {
1399         gni_return_t rrc;
1400
1401         /* Error injection */
1402         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_MAP)) {
1403                 rrc = GNI_RC_ERROR_RESOURCE;
1404         } else if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_SMALL_MAP) &&
1405                    length <= *kgnilnd_tunables.kgn_max_immediate) {
1406                 rrc = GNI_RC_INVALID_PARAM;
1407         } else {
1408                 rrc = gni_mem_register(
1409                         nic_hndl, address, length,
1410                         dst_cq_hndl, flags, mem_hndl);
1411         }
1412
1413         switch (rrc)  {
1414         case GNI_RC_SUCCESS:
1415         case GNI_RC_ERROR_RESOURCE:
1416                 break;
1417         case GNI_RC_INVALID_PARAM:
1418                 GNILND_API_SWBUG(
1419                         nic_hndl, address, length,
1420                         dst_cq_hndl, flags, mem_hndl);
1421                 break;
1422         default:
1423                 GNILND_API_RC_LBUG(
1424                         nic_hndl, address, length,
1425                         dst_cq_hndl, flags, mem_hndl);
1426
1427                 /* LBUG never returns, but just for style and consistency */
1428                 break;
1429         }
1430         RETURN(rrc);
1431 }
1432 #undef apick_fn
1433 #undef apick_fmt
1434
1435 #define apick_fn "kgnilnd_mem_deregister"
1436 #define apick_fmt "0x%p,0x%p,%d"
1437 static inline gni_return_t kgnilnd_mem_deregister(
1438                 IN gni_nic_handle_t     nic_hndl,
1439                 IN gni_mem_handle_t     *mem_hndl,
1440                 IN int                  hold_timeout
1441                 )
1442 {
1443         gni_return_t rrc;
1444
1445         /* Error injection */
1446         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_UNMAP)) {
1447                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
1448         } else {
1449                 rrc = gni_mem_deregister(
1450                         nic_hndl, mem_hndl, hold_timeout);
1451         }
1452
1453         switch (rrc)  {
1454         case GNI_RC_SUCCESS:
1455                 break;
1456         case GNI_RC_INVALID_PARAM:
1457                 GNILND_API_SWBUG(
1458                         nic_hndl, mem_hndl, hold_timeout);
1459                 break;
1460         default:
1461                 GNILND_API_RC_LBUG(
1462                         nic_hndl, mem_hndl, hold_timeout);
1463
1464                 /* LBUG never returns, but just for style and consistency */
1465                 break;
1466         }
1467         RETURN(rrc);
1468 }
1469 #undef apick_fn
1470 #undef apick_fmt
1471
1472 #define apick_fn "kgnilnd_mem_mdd_release"
1473 #define apick_fmt "0x%p,0x%p"
1474 static inline gni_return_t kgnilnd_mem_mdd_release(
1475                 IN gni_nic_handle_t     nic_hndl,
1476                 IN gni_mem_handle_t     *mem_hndl
1477                 )
1478 {
1479         gni_return_t rrc;
1480
1481         /* Error injection */
1482         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_MDD_RELEASE)) {
1483                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NO_MATCH;
1484         } else {
1485                 rrc = gni_mem_mdd_release(
1486                         nic_hndl, mem_hndl);
1487         }
1488
1489         switch (rrc)  {
1490         case GNI_RC_SUCCESS:
1491         case GNI_RC_NO_MATCH:
1492                 break;
1493         default:
1494                 GNILND_API_RC_LBUG(
1495                         nic_hndl, mem_hndl);
1496
1497                 /* LBUG never returns, but just for style and consistency */
1498                 break;
1499         }
1500         RETURN(rrc);
1501 }
1502 #undef apick_fn
1503 #undef apick_fmt
1504
1505 #endif /* _GNILND_API_WRAP_H */