lnet/klnds/gnilnd/gnilnd_api_wrap.h

   1 /*
   2  * Copyright (C) 2009-2012 Cray, Inc.
   3  *
   4  *   Author: Nic Henke <nic@cray.com>
   5  *
   6  *   This file is part of Lustre, http://www.lustre.org.
   7  *
   8  *   Lustre is free software; you can redistribute it and/or
   9  *   modify it under the terms of version 2 of the GNU General Public
  10  *   License as published by the Free Software Foundation.
  11  *
  12  *   Lustre is distributed in the hope that it will be useful,
  13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *   GNU General Public License for more details.
  16  *
  17  *   You should have received a copy of the GNU General Public License
  18  *   along with Lustre; if not, write to the Free Software
  19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20  *
  21  */
  22 #ifndef _GNILND_API_WRAP_H
  23 #define _GNILND_API_WRAP_H
  24
  25 /* LNet is allocated failure locations 0xe000 to 0xffff */
  26
  27 /* GNILND has 0xf0XX */
  28 #define CFS_FAIL_GNI                    0xf000
  29 #define CFS_FAIL_GNI_PHYS_MAP           0xf001
  30 #define CFS_FAIL_GNI_VIRT_MAP           0xf002
  31 #define CFS_FAIL_GNI_GET_UNMAP          0xf003
  32 #define CFS_FAIL_GNI_PUT_UNMAP          0xf004
  33 #define CFS_FAIL_GNI_MAP_TX             0xf005
  34 #define CFS_FAIL_GNI_SMSG_SEND          0xf006
  35 #define CFS_FAIL_GNI_CLOSE_SEND         0xf007
  36 #define CFS_FAIL_GNI_CDM_CREATE         0xf008
  37 #define CFS_FAIL_GNI_CDM_DESTROY        0xf009
  38 #define CFS_FAIL_GNI_CDM_ATTACH         0xf00a
  39 #define CFS_FAIL_GNI_CQ_CREATE          0xf00b
  40 #define CFS_FAIL_GNI_CQ_DESTROY         0xf00c
  41 #define CFS_FAIL_GNI_EP_BIND            0xf00d
  42 #define CFS_FAIL_GNI_EP_UNBIND          0xf00e
  43 #define CFS_FAIL_GNI_EP_SET_EVDATA      0xf00f
  44 #define CFS_FAIL_GNI_SMSG_INIT          0xf010
  45 #define CFS_FAIL_GNI_SMSG_RELEASE       0xf011
  46 #define CFS_FAIL_GNI_POST_RDMA          0xf012
  47 #define CFS_FAIL_GNI_GET_COMPLETED      0xf013
  48 #define CFS_FAIL_GNI_EP_DESTROY         0xf015
  49 #define CFS_FAIL_GNI_VIRT_UNMAP         0xf016
  50 #define CFS_FAIL_GNI_MDD_RELEASE        0xf017
  51 #define CFS_FAIL_GNI_NOOP_SEND          0xf018
  52 #define CFS_FAIL_GNI_ERR_SUBSCRIBE      0xf01a
  53 #define CFS_FAIL_GNI_QUIESCE_RACE       0xf01b
  54 #define CFS_FAIL_GNI_DG_TERMINATE       0xf01c
  55 #define CFS_FAIL_GNI_REG_QUIESCE        0xf01d
  56 #define CFS_FAIL_GNI_IN_QUIESCE         0xf01e
  57 #define CFS_FAIL_GNI_DELAY_RDMA         0xf01f
  58 #define CFS_FAIL_GNI_SR_DOWN_RACE       0xf020
  59 #define CFS_FAIL_GNI_ALLOC_TX           0xf021
  60 #define CFS_FAIL_GNI_FMABLK_AVAIL       0xf022
  61 #define CFS_FAIL_GNI_EP_CREATE          0xf023
  62 #define CFS_FAIL_GNI_CQ_GET_EVENT       0xf024
  63 #define CFS_FAIL_GNI_PROBE              0xf025
  64 #define CFS_FAIL_GNI_EP_TEST            0xf026
  65 #define CFS_FAIL_GNI_CONNREQ_DROP       0xf027
  66 #define CFS_FAIL_GNI_CONNREQ_PROTO      0xf028
  67 #define CFS_FAIL_GNI_CONND_PILEUP       0xf029
  68 #define CFS_FAIL_GNI_PHYS_SETUP         0xf02a
  69 #define CFS_FAIL_GNI_FIND_TARGET        0xf02b
  70 #define CFS_FAIL_GNI_WC_DGRAM_FREE      0xf02c
  71 #define CFS_FAIL_GNI_DROP_CLOSING       0xf02d
  72 #define CFS_FAIL_GNI_RX_CLOSE_CLOSING   0xf02e
  73 #define CFS_FAIL_GNI_RX_CLOSE_CLOSED    0xf02f
  74 #define CFS_FAIL_GNI_EP_POST            0xf030
  75 #define CFS_FAIL_GNI_PACK_SRCNID        0xf031
  76 #define CFS_FAIL_GNI_PACK_DSTNID        0xf032
  77 #define CFS_FAIL_GNI_PROBE_WAIT         0xf033
  78 #define CFS_FAIL_GNI_SMSG_CKSUM1        0xf034
  79 #define CFS_FAIL_GNI_SMSG_CKSUM2        0xf035
  80 #define CFS_FAIL_GNI_SMSG_CKSUM3        0xf036
  81 #define CFS_FAIL_GNI_DROP_DESTROY_EP    0xf037
  82 #define CFS_FAIL_GNI_SMSG_GETNEXT       0xf038
  83 #define CFS_FAIL_GNI_FINISH_PURG        0xf039
  84 #define CFS_FAIL_GNI_PURG_REL_DELAY     0xf03a
  85 #define CFS_FAIL_GNI_DONT_NOTIFY        0xf03b
  86 #define CFS_FAIL_GNI_VIRT_SMALL_MAP     0xf03c
  87 #define CFS_FAIL_GNI_DELAY_RDMAQ        0xf03d
  88 #define CFS_FAIL_GNI_PAUSE_SHUTDOWN     0xf03e
  89 #define CFS_FAIL_GNI_PAUSE_DGRAM_COMP   0xf03f
  90 #define CFS_FAIL_GNI_NET_LOOKUP         0xf040
  91 #define CFS_FAIL_GNI_RECV_TIMEOUT       0xf041
  92 #define CFS_FAIL_GNI_SEND_TIMEOUT       0xf042
  93 #define CFS_FAIL_GNI_ONLY_NOOP          0xf043
  94 #define CFS_FAIL_GNI_FINISH_PURG2       0xf044
  95 #define CFS_FAIL_GNI_RACE_RESET         0xf045
  96 #define CFS_FAIL_GNI_GNP_CONNECTING1    0xf046
  97 #define CFS_FAIL_GNI_GNP_CONNECTING2    0xf047
  98 #define CFS_FAIL_GNI_GNP_CONNECTING3    0xf048
  99 #define CFS_FAIL_GNI_SCHEDULE_COMPLETE  0xf049
 100 #define CFS_FAIL_GNI_PUT_ACK_AGAIN      0xf050
 101 #define CFS_FAIL_GNI_GET_REQ_AGAIN      0xf051
 102 #define CFS_FAIL_GNI_SCHED_DEADLINE     0xf052
 103 #define CFS_FAIL_GNI_DGRAM_DEADLINE     0xf053
 104 #define CFS_FAIL_GNI_DGRAM_DROP_TX      0xf054
 105
 106 extern uint32_t kgni_driver_version;
 107
 108 static inline void
 109 kgnilnd_check_kgni_version(void)
 110 {
 111         uint32_t *kdv;
 112
 113         kgnilnd_data.kgn_enable_gl_mutex = 1;
 114         kdv = symbol_get(kgni_driver_version);
 115         if (!kdv) {
 116                 LCONSOLE_INFO("Not using thread safe locking -"
 117                         " no symbol kgni_driver_version\n");
 118                 return;
 119         }
 120
 121         /* Thread-safe kgni implemented in minor ver 0x44/45, code rev 0xb9 */
 122         if (*kdv < GNI_VERSION_CHECK(0, GNILND_KGNI_TS_MINOR_VER, 0xb9)) {
 123                 symbol_put(kgni_driver_version);
 124                 LCONSOLE_INFO("Not using thread safe locking, gni version 0x%x,"
 125                         " need >= 0x%x\n", *kdv,
 126                         GNI_VERSION_CHECK(0, GNILND_KGNI_TS_MINOR_VER, 0xb9));
 127                 return;
 128         }
 129
 130         symbol_put(kgni_driver_version);
 131
 132         if (!*kgnilnd_tunables.kgn_thread_safe) {
 133                 return;
 134         }
 135
 136         /* Use thread-safe locking */
 137         kgnilnd_data.kgn_enable_gl_mutex = 0;
 138 }
 139
 140 /* helper macros */
 141 extern void
 142 _kgnilnd_api_rc_lbug(const char *rcstr, int rc, struct libcfs_debug_msg_data *data,
 143                         const char *fmt, ...)
 144         __attribute__ ((format (printf, 4, 5)));
 145
 146 #define kgnilnd_api_rc_lbug(msgdata, rc, fmt, a...)                             \
 147 do {                                                                            \
 148         CFS_CHECK_STACK(msgdata, D_ERROR, NULL);                                \
 149         /* we don't mask this - it is always at D_ERROR */                      \
 150         _kgnilnd_api_rc_lbug(kgnilnd_api_rc2str(rc), (rc), msgdata, fmt, ##a);  \
 151 } while (0)
 152
 153 #define DO_RETCODE(x) case x: return #x;
 154 static inline const char *
 155 kgnilnd_api_rc2str(gni_return_t rrc)
 156 {
 157
 158         switch (rrc) {
 159                 DO_RETCODE(GNI_RC_SUCCESS)
 160                 DO_RETCODE(GNI_RC_NOT_DONE);
 161                 DO_RETCODE(GNI_RC_INVALID_PARAM);
 162                 DO_RETCODE(GNI_RC_ERROR_RESOURCE);
 163                 DO_RETCODE(GNI_RC_TIMEOUT);
 164                 DO_RETCODE(GNI_RC_PERMISSION_ERROR);
 165                 DO_RETCODE(GNI_RC_DESCRIPTOR_ERROR);
 166                 DO_RETCODE(GNI_RC_ALIGNMENT_ERROR);
 167                 DO_RETCODE(GNI_RC_INVALID_STATE);
 168                 DO_RETCODE(GNI_RC_NO_MATCH);
 169                 DO_RETCODE(GNI_RC_SIZE_ERROR);
 170                 DO_RETCODE(GNI_RC_TRANSACTION_ERROR);
 171                 DO_RETCODE(GNI_RC_ILLEGAL_OP);
 172                 DO_RETCODE(GNI_RC_ERROR_NOMEM);
 173         }
 174         LBUG();
 175 }
 176 #undef DO_RETCODE
 177
 178 /* log an error and LBUG for unhandled rc from gni api function
 179  * the fmt should be something like:
 180  *  gni_api_call(arg1, arg2, arg3)
 181  */
 182
 183 /* apick_fn and apick_fmt should be defined for each site */
 184 #undef apick_fn
 185 #undef apick_fmt
 186
 187 #define GNILND_API_RC_LBUG(args...)                                             \
 188 do {                                                                            \
 189         LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);                     \
 190         kgnilnd_api_rc_lbug(&msgdata, rrc, apick_fn"("apick_fmt")", ##args);    \
 191 } while (0)
 192
 193 #define GNILND_API_SWBUG(args...)                                               \
 194 do {                                                                            \
 195         CERROR("likely SOFTWARE BUG "apick_fn"("apick_fmt") rc %s\n",           \
 196                  ##args, kgnilnd_api_rc2str(rrc));                              \
 197 } while (0)
 198
 199 #define GNILND_API_EINVAL(args...)                                              \
 200 do {                                                                            \
 201         CERROR("invalid parameter to "apick_fn"("apick_fmt") rc %s\n",          \
 202                  ##args, kgnilnd_api_rc2str(rrc));                              \
 203 } while (0)
 204
 205 #define GNILND_API_RESOURCE(args...)                                            \
 206 do {                                                                            \
 207         CERROR("no resources for "apick_fn"("apick_fmt") rc %s\n",              \
 208                 ##args, kgnilnd_api_rc2str(rrc));                               \
 209 } while (0)
 210
 211 #define GNILND_API_BUSY(args...)                                                \
 212 do {                                                                            \
 213         CERROR("resources busy for "apick_fn"("apick_fmt") rc %s\n",            \
 214                 ##args, kgnilnd_api_rc2str(rrc));                               \
 215 } while (0)
 216
 217 #undef DEBUG_SMSG_CREDITS
 218 #ifdef DEBUG_SMSG_CREDITS
 219 #define CRAY_CONFIG_GHAL_GEMINI
 220 #include <gni_priv.h>
 221 #define GNIDBG_SMSG_CREDS(level, conn)                                        \
 222 do {                                                                          \
 223         gni_ep_smsg_mbox_t *smsg = conn->gnc_ephandle->smsg;                  \
 224         CDEBUG(level, "SMSGDBG: conn %p mcred %d/%d bcred %d/%d "             \
 225                 "s_seq %d/%d/%d r_seq %d/%d/%d retr %d\n",                    \
 226                 conn, smsg->mbox_credits, smsg->back_mbox_credits,            \
 227                 smsg->buffer_credits, smsg->back_buffer_credits,              \
 228                 smsg->s_seqno, smsg->s_seqno_back_mbox_credits,               \
 229                 smsg->s_seqno_back_buffer_credits, smsg->r_seqno,             \
 230                 smsg->r_seqno_back_mbox_credits,                              \
 231                 smsg->r_seqno_back_buffer_credits, smsg->retransmit_count);   \
 232 } while (0)
 233 #else
 234 #define GNIDBG_SMSG_CREDS(level, conn) do {} while(0)
 235 #endif
 236
 237 /* these are all wrappers around gni_XXX functions.
 238  * This allows us to handle all the return codes and api checks without
 239  * dirtying up the logic code */
 240
 241 /* TODO: RETURN wrapper that translates integer to GNI API RC string */
 242
 243 #define apick_fn "kgnilnd_cdm_create"
 244 #define apick_fmt "%u, %u, %u, %u, 0x%p"
 245 static inline gni_return_t kgnilnd_cdm_create(
 246                 IN uint32_t             inst_id,
 247                 IN uint8_t              ptag,
 248                 IN uint32_t             cookie,
 249                 IN uint32_t             modes,
 250                 OUT gni_cdm_handle_t    *cdm_hndl
 251                 )
 252 {
 253         gni_return_t rrc;
 254
 255         /* Error injection */
 256         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CDM_CREATE)) {
 257                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 258         } else {
 259                 rrc = gni_cdm_create(inst_id, ptag, cookie, modes, cdm_hndl);
 260         }
 261
 262         switch (rrc)  {
 263         case GNI_RC_SUCCESS:
 264                 break;
 265         case GNI_RC_ERROR_RESOURCE:
 266         case GNI_RC_INVALID_PARAM:
 267                 /* Try to bail gracefully */
 268                 GNILND_API_SWBUG(
 269                         inst_id, ptag, cookie, modes, cdm_hndl);
 270                 break;
 271         default:
 272                 GNILND_API_RC_LBUG(
 273                         inst_id, ptag, cookie, modes, cdm_hndl);
 274
 275                 /* LBUG never returns, but just for style and consistency */
 276                 break;
 277         }
 278         RETURN(rrc);
 279 }
 280
 281 #undef apick_fn
 282 #undef apick_fmt
 283
 284 #define apick_fn "kgnilnd_cdm_attach"
 285 #define apick_fmt "0x%p, %u, 0x%p, 0x%p"
 286 static inline gni_return_t kgnilnd_cdm_attach(
 287                 IN gni_cdm_handle_t     cdm_hndl,
 288                 IN uint32_t             device_id,
 289                 OUT uint32_t            *local_addr,
 290                 OUT gni_nic_handle_t    *nic_hndl
 291                 )
 292 {
 293         gni_return_t rrc;
 294
 295         /* Error injection */
 296         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CDM_ATTACH)) {
 297                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 298         } else {
 299                 rrc = gni_cdm_attach(cdm_hndl, device_id, local_addr, nic_hndl);
 300         }
 301
 302         switch (rrc)  {
 303         case GNI_RC_SUCCESS:
 304                 break;
 305         case GNI_RC_NO_MATCH:
 306         case GNI_RC_INVALID_PARAM:
 307                 GNILND_API_SWBUG(
 308                         cdm_hndl, device_id, local_addr, nic_hndl);
 309                 break;
 310         case GNI_RC_ERROR_RESOURCE:
 311         case GNI_RC_INVALID_STATE:
 312                 GNILND_API_RESOURCE(
 313                         cdm_hndl, device_id, local_addr, nic_hndl);
 314                 break;
 315         default:
 316                 GNILND_API_RC_LBUG(
 317                         cdm_hndl, device_id, local_addr, nic_hndl);
 318
 319                 /* LBUG never returns, but just for style and consistency */
 320                 break;
 321         }
 322         RETURN(rrc);
 323 }
 324 #undef apick_fmt
 325 #undef apick_fn
 326
 327 #define apick_fn "kgnilnd_cdm_destroy"
 328 #define apick_fmt "0x%p"
 329 static inline gni_return_t kgnilnd_cdm_destroy(
 330                 IN gni_cdm_handle_t     cdm_hndl
 331                 )
 332 {
 333         gni_return_t rrc;
 334
 335         /* Error injection */
 336         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_DESTROY)) {
 337                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 338         } else {
 339                 rrc = gni_cdm_destroy(
 340                         cdm_hndl);
 341         }
 342
 343         switch (rrc)  {
 344         case GNI_RC_SUCCESS:
 345                 break;
 346         case GNI_RC_INVALID_PARAM:
 347                 GNILND_API_SWBUG(
 348                         cdm_hndl);
 349                 break;
 350         default:
 351                 GNILND_API_RC_LBUG(
 352                         cdm_hndl);
 353
 354                 /* LBUG never returns, but just for style and consistency */
 355                 break;
 356         }
 357         RETURN(rrc);
 358 }
 359 #undef apick_fn
 360 #undef apick_fmt
 361
 362 #define apick_fn "kgnilnd_subscribe_errors"
 363 #define apick_fmt "0x%p,%x,%u,0x%p,0x%p,0x%p"
 364 static inline gni_return_t kgnilnd_subscribe_errors(
 365                 IN gni_nic_handle_t  nic_handle,
 366                 IN gni_error_mask_t  mask,
 367                 IN uint32_t          EEQ_size,
 368                 IN void              (*EQ_new_event)(gni_err_handle_t),
 369                 IN void              (*app_crit_err)(gni_err_handle_t),
 370                 OUT gni_err_handle_t *err_handle
 371                 )
 372 {
 373         gni_return_t rrc;
 374
 375         /* Error injection */
 376         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_ERR_SUBSCRIBE)) {
 377                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 378         } else {
 379                 rrc = gni_subscribe_errors(
 380                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 381                         err_handle);
 382         }
 383
 384         switch (rrc)  {
 385         case GNI_RC_SUCCESS:
 386                 break;
 387         case GNI_RC_INVALID_PARAM:
 388                 GNILND_API_SWBUG(
 389                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 390                         err_handle);
 391                 break;
 392         case GNI_RC_ERROR_RESOURCE:
 393                 GNILND_API_RESOURCE(
 394                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 395                         err_handle);
 396                 break;
 397         default:
 398                 GNILND_API_RC_LBUG(
 399                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 400                         err_handle);
 401                 /* LBUG never returns, but just for style and consistency */
 402                 break;
 403         }
 404         RETURN(rrc);
 405 }
 406 #undef apick_fn
 407 #undef apick_fmt
 408
 409 #define apick_fn "kgnilnd_release_errors"
 410 #define apick_fmt "0x%p"
 411 static inline gni_return_t kgnilnd_release_errors(
 412                 IN gni_err_handle_t err_handle
 413                 )
 414 {
 415         gni_return_t rrc;
 416
 417         rrc = gni_release_errors(
 418                         err_handle);
 419
 420         switch (rrc)  {
 421         case GNI_RC_SUCCESS:
 422                 break;
 423         case GNI_RC_INVALID_PARAM:
 424         case GNI_RC_NOT_DONE:
 425                 GNILND_API_SWBUG(
 426                         err_handle);
 427                 break;
 428         default:
 429                 GNILND_API_RC_LBUG(
 430                         err_handle);
 431                 /* LBUG never returns, but just for style and consistency */
 432                 break;
 433         }
 434         RETURN(rrc);
 435 }
 436 #undef apick_fn
 437 #undef apick_fmt
 438
 439 #define apick_fn "kgnilnd_set_quiesce_callback"
 440 #define apick_fmt "0x%p,0x%p"
 441 static inline gni_return_t kgnilnd_set_quiesce_callback(
 442                 IN gni_nic_handle_t  nic_handle,
 443                 IN void              (*qsce_func)(gni_nic_handle_t, uint64_t msecs)
 444                 )
 445 {
 446         gni_return_t rrc;
 447
 448         /* Error injection */
 449         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_REG_QUIESCE)) {
 450                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 451         } else {
 452                 rrc = gni_set_quiesce_callback(
 453                         nic_handle, qsce_func);
 454         }
 455
 456         switch (rrc)  {
 457         case GNI_RC_SUCCESS:
 458                 break;
 459         case GNI_RC_INVALID_STATE:
 460         case GNI_RC_INVALID_PARAM:
 461                 GNILND_API_SWBUG(
 462                         nic_handle, qsce_func);
 463                 break;
 464         default:
 465                 GNILND_API_RC_LBUG(
 466                         nic_handle, qsce_func);
 467                 /* LBUG never returns, but just for style and consistency */
 468                 break;
 469         }
 470         RETURN(rrc);
 471 }
 472 #undef apick_fn
 473 #undef apick_fmt
 474
 475 #define apick_fn "kgnilnd_get_quiesce_status"
 476 #define apick_fmt "0x%p"
 477 static inline gni_return_t kgnilnd_get_quiesce_status(
 478                 IN gni_nic_handle_t  nic_handle
 479                 )
 480 {
 481         uint32_t rrc;
 482
 483         /* this has weird RC -
 484          * 0 - quiesce not in progress
 485          * 1 - quiesce is turned on
 486         */
 487
 488         /* Error injection */
 489         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_IN_QUIESCE)) {
 490                 rrc = 1;
 491         } else {
 492                 rrc = gni_get_quiesce_status(
 493                         nic_handle);
 494         }
 495
 496         switch (rrc)  {
 497         case 1:
 498         case 0:
 499                 break;
 500         default:
 501                 GNILND_API_RC_LBUG(
 502                         nic_handle);
 503                 /* LBUG never returns, but just for style and consistency */
 504                 break;
 505         }
 506         RETURN(rrc);
 507 }
 508 #undef apick_fn
 509 #undef apick_fmt
 510
 511 #define apick_fn "kgnilnd_cq_create"
 512 #define apick_fmt "0x%p, %u, %u, 0x%p, "LPX64", 0x%p"
 513 static inline gni_return_t kgnilnd_cq_create(
 514                 IN gni_nic_handle_t     nic_hndl,
 515                 IN uint32_t             entry_count,
 516                 IN uint32_t             delay_index,
 517                 IN gni_cq_event_hndlr_f *event_handler,
 518                 IN uint64_t             usr_event_data,
 519                 OUT gni_cq_handle_t     *cq_hndl
 520                 )
 521 {
 522         gni_return_t rrc;
 523
 524         /* Error injection */
 525         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_CREATE)) {
 526                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 527         } else {
 528                 rrc = gni_cq_create(
 529                        nic_hndl, entry_count, delay_index, event_handler,
 530                         usr_event_data, cq_hndl);
 531         }
 532
 533         switch (rrc)  {
 534         case GNI_RC_SUCCESS:
 535                 break;
 536         case GNI_RC_INVALID_PARAM:
 537                 GNILND_API_SWBUG(
 538                         nic_hndl, entry_count, delay_index, event_handler,
 539                         usr_event_data, cq_hndl);
 540                 break;
 541         case GNI_RC_ERROR_RESOURCE:
 542                 GNILND_API_RESOURCE(
 543                         nic_hndl, entry_count, delay_index, event_handler,
 544                         usr_event_data, cq_hndl);
 545                 break;
 546         default:
 547                 GNILND_API_RC_LBUG(
 548                         nic_hndl, entry_count, delay_index, event_handler,
 549                         usr_event_data, cq_hndl);
 550
 551                 /* LBUG never returns, but just for style and consistency */
 552                 break;
 553         }
 554         RETURN(rrc);
 555 }
 556 #undef apick_fn
 557 #undef apick_fmt
 558
 559 #define apick_fn "kgnilnd_cq_destroy"
 560 #define apick_fmt "0x%p"
 561 static inline gni_return_t kgnilnd_cq_destroy(
 562                 IN gni_cq_handle_t cq_hndl
 563                 )
 564 {
 565         gni_return_t rrc;
 566
 567         /* Error injection */
 568         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_DESTROY)) {
 569                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 570         } else {
 571
 572                 rrc = gni_cq_destroy(
 573                         cq_hndl);
 574         }
 575
 576         switch (rrc)  {
 577         case GNI_RC_SUCCESS:
 578                 break;
 579         case GNI_RC_INVALID_PARAM:
 580                 GNILND_API_SWBUG(
 581                         cq_hndl);
 582                 break;
 583         case GNI_RC_ERROR_RESOURCE:
 584                 GNILND_API_BUSY(
 585                         cq_hndl);
 586                 break;
 587         default:
 588                 GNILND_API_RC_LBUG(
 589                         cq_hndl);
 590
 591                 /* LBUG never returns, but just for style and consistency */
 592                 break;
 593         }
 594         RETURN(rrc);
 595 }
 596 #undef apick_fn
 597 #undef apick_fmt
 598
 599 #define apick_fn "kgnilnd_cq_get_event"
 600 #define apick_fmt "0x%p, 0x%p"
 601 static inline gni_return_t kgnilnd_cq_get_event(
 602                 IN gni_cq_handle_t cq_hndl,
 603                 OUT gni_cq_entry_t *event_data
 604                 )
 605 {
 606         gni_return_t rrc;
 607
 608         /* no error injection - CQs are touchy about the data.
 609          * where appropriate, we'll do this on the CQs that should be able to
 610          * handle the various errors */
 611         rrc = gni_cq_get_event(
 612                         cq_hndl, event_data);
 613
 614         switch (rrc)  {
 615         case GNI_RC_SUCCESS:
 616         case GNI_RC_NOT_DONE:
 617         case GNI_RC_TRANSACTION_ERROR:
 618                 break;
 619         case GNI_RC_ERROR_RESOURCE:
 620                 LASSERTF(GNI_CQ_OVERRUN(*event_data),
 621                          "kgni returned ERROR_RESOURCE but cq_hndl 0x%p is not "
 622                          "overrun\n", cq_hndl);
 623                 break;
 624         case GNI_RC_INVALID_PARAM:
 625                 GNILND_API_SWBUG(
 626                         cq_hndl, event_data);
 627                 break;
 628         default:
 629                 GNILND_API_RC_LBUG(
 630                         cq_hndl, event_data);
 631
 632                 /* LBUG never returns, but just for style and consistency */
 633                 break;
 634         }
 635         return rrc;
 636 }
 637 #undef apick_fn
 638 #undef apick_fmt
 639
 640 #define apick_fn "kgnilnd_smsg_init"
 641 #define apick_fmt "0x%p, 0x%p, 0x%p"
 642 static inline gni_return_t kgnilnd_smsg_init(
 643                 IN gni_ep_handle_t      ep_hndl,
 644                 IN gni_smsg_attr_t      *local_smsg_attr,
 645                 IN gni_smsg_attr_t      *remote_smsg_attr
 646                 )
 647 {
 648         gni_return_t rrc;
 649
 650         /* Error injection */
 651         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_INIT)) {
 652                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_RESOURCE;
 653         } else {
 654                 rrc = gni_smsg_init(
 655                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 656         }
 657
 658         switch (rrc)  {
 659         /* both of these are OK, upper SW needs to handle */
 660         case GNI_RC_SUCCESS:
 661         case GNI_RC_NOT_DONE:
 662                 break;
 663         case GNI_RC_INVALID_PARAM:
 664         case GNI_RC_INVALID_STATE:
 665                 GNILND_API_SWBUG(
 666                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 667                 break;
 668         case GNI_RC_ERROR_RESOURCE:
 669                 GNILND_API_RESOURCE(
 670                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 671                 break;
 672         default:
 673                 GNILND_API_RC_LBUG(
 674                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 675
 676                 /* LBUG never returns, but just for style and consistency */
 677                 break;
 678         }
 679         RETURN(rrc);
 680 }
 681 #undef apick_fn
 682 #undef apick_fmt
 683
 684 #define apick_fn "kgnilnd_smsg_send"
 685 #define apick_fmt "0x%p, 0x%p, %d, 0x%p, %u %u"
 686 static inline gni_return_t kgnilnd_smsg_send(
 687                 IN gni_ep_handle_t      ep_hndl,
 688                 IN void                 *header,
 689                 IN uint32_t             header_length,
 690                 IN void                 *data,
 691                 IN uint32_t             data_length,
 692                 IN uint32_t             msg_id
 693                 )
 694 {
 695         gni_return_t rrc;
 696
 697         /* Error injection */
 698         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_SEND)) {
 699                 if (cfs_fail_loc & CFS_FAIL_RAND) {
 700                         rrc = GNI_RC_NOT_DONE;
 701                 } else {
 702                         rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 703                 }
 704         } else {
 705                 rrc = gni_smsg_send(
 706                         ep_hndl, header, header_length, data, data_length, msg_id);
 707         }
 708
 709         switch (rrc)  {
 710         /* both of these are OK, upper SW needs to handle */
 711         case GNI_RC_SUCCESS:
 712         case GNI_RC_NOT_DONE:
 713                 break;
 714         case GNI_RC_INVALID_PARAM:
 715                 GNILND_API_SWBUG(
 716                         ep_hndl, header, header_length, data, data_length, msg_id);
 717                 break;
 718         case GNI_RC_ERROR_RESOURCE:
 719                 GNILND_API_RESOURCE(
 720                         ep_hndl, header, header_length, data, data_length, msg_id);
 721                 break;
 722         default:
 723                 GNILND_API_RC_LBUG(
 724                         ep_hndl, header, header_length, data, data_length, msg_id);
 725
 726                 /* LBUG never returns, but just for style and consistency */
 727                 break;
 728         }
 729         RETURN(rrc);
 730 }
 731 #undef apick_fn
 732 #undef apick_fmt
 733
 734 #define apick_fn "kgnilnd_smsg_getnext"
 735 #define apick_fmt "0x%p,0x%p"
 736 static inline gni_return_t kgnilnd_smsg_getnext(
 737                 IN gni_ep_handle_t      ep_hndl,
 738                 OUT void                **header
 739                 )
 740 {
 741         gni_return_t rrc;
 742
 743         /* Error injection */
 744         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_RELEASE)) {
 745                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 746         } else {
 747                 rrc = gni_smsg_getnext(
 748                         ep_hndl, header);
 749         }
 750
 751         switch (rrc)  {
 752         /* both of these are OK, upper SW needs to handle */
 753         case GNI_RC_SUCCESS:
 754         case GNI_RC_NOT_DONE:
 755         case GNI_RC_INVALID_STATE:
 756                 break;
 757         case GNI_RC_INVALID_PARAM:
 758                 GNILND_API_SWBUG(
 759                         ep_hndl, header);
 760                 break;
 761         default:
 762                 GNILND_API_RC_LBUG(
 763                         ep_hndl, header);
 764
 765                 /* LBUG never returns, but just for style and consistency */
 766                 break;
 767         }
 768         RETURN(rrc);
 769 }
 770 #undef apick_fn
 771 #undef apick_fmt
 772
 773 #define apick_fn "kgnilnd_smsg_release"
 774 #define apick_fmt "0x%p"
 775 static inline gni_return_t kgnilnd_smsg_release(
 776                 IN gni_ep_handle_t      ep_hndl
 777                 )
 778 {
 779         gni_return_t rrc;
 780
 781         /* Error injection */
 782         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_RELEASE)) {
 783                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 784         } else {
 785                 rrc = gni_smsg_release(
 786                         ep_hndl);
 787         }
 788
 789         switch (rrc)  {
 790         /* both of these are OK, upper SW needs to handle */
 791         case GNI_RC_SUCCESS:
 792         case GNI_RC_NOT_DONE:
 793                 break;
 794         case GNI_RC_INVALID_PARAM:
 795                 GNILND_API_SWBUG(
 796                         ep_hndl);
 797                 break;
 798         default:
 799                 GNILND_API_RC_LBUG(
 800                         ep_hndl);
 801
 802                 /* LBUG never returns, but just for style and consistency */
 803                 break;
 804         }
 805         RETURN(rrc);
 806 }
 807 #undef apick_fn
 808 #undef apick_fmt
 809
 810 #define apick_fn "kgnilnd_ep_create"
 811 #define apick_fmt "0x%p, 0x%p, 0x%p"
 812 static inline gni_return_t kgnilnd_ep_create(
 813                 IN gni_nic_handle_t     nic_hndl,
 814                 IN gni_cq_handle_t      src_cq_hndl,
 815                 OUT gni_ep_handle_t     *ep_hndl
 816                 )
 817 {
 818         gni_return_t rrc;
 819
 820         /* error injection */
 821         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_CREATE)) {
 822                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_NOMEM;
 823         } else {
 824                 rrc = gni_ep_create(
 825                         nic_hndl, src_cq_hndl, ep_hndl);
 826         }
 827
 828         switch (rrc)  {
 829         case GNI_RC_SUCCESS:
 830                 break;
 831         case GNI_RC_INVALID_PARAM:
 832                 GNILND_API_SWBUG(
 833                         nic_hndl, src_cq_hndl, ep_hndl);
 834                 break;
 835         case GNI_RC_ERROR_NOMEM:
 836                 GNILND_API_RESOURCE(
 837                         nic_hndl, src_cq_hndl, ep_hndl);
 838                 break;
 839         default:
 840                 GNILND_API_RC_LBUG(
 841                         nic_hndl, src_cq_hndl, ep_hndl);
 842
 843                 /* lbug never returns, but just for style and consistency */
 844                 break;
 845         }
 846         RETURN(rrc);
 847 }
 848 #undef apick_fn
 849 #undef apick_fmt
 850
 851 #define apick_fn "kgnilnd_ep_bind"
 852 #define apick_fmt "0x%p, %x, %x"
 853 static inline gni_return_t kgnilnd_ep_bind(
 854                 IN gni_ep_handle_t      ep_hndl,
 855                 IN uint32_t             remote_addr,
 856                 IN uint32_t             remote_id
 857                 )
 858 {
 859         gni_return_t rrc;
 860
 861         /* error injection */
 862         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_BIND)) {
 863                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 864         } else {
 865                 rrc = gni_ep_bind(
 866                         ep_hndl, remote_addr, remote_id);
 867         }
 868
 869         switch (rrc)  {
 870         /* both of these are ok, upper sw needs to handle */
 871         case GNI_RC_SUCCESS:
 872         case GNI_RC_NOT_DONE:
 873                 break;
 874         case GNI_RC_INVALID_PARAM:
 875                 GNILND_API_SWBUG(
 876                         ep_hndl, remote_addr, remote_id);
 877                 break;
 878         default:
 879                 GNILND_API_RC_LBUG(
 880                         ep_hndl, remote_addr, remote_id);
 881
 882                 /* lbug never returns, but just for style and consistency */
 883                 break;
 884         }
 885         RETURN(rrc);
 886 }
 887 #undef apick_fn
 888 #undef apick_fmt
 889
 890 #define apick_fn "kgnilnd_ep_set_eventdata"
 891 #define apick_fmt "0x%p, %x, %x"
 892 static inline gni_return_t kgnilnd_ep_set_eventdata(
 893                 IN gni_ep_handle_t      ep_hndl,
 894                 IN uint32_t             local_event,
 895                 IN uint32_t             remote_event
 896                 )
 897 {
 898         gni_return_t rrc;
 899
 900         /* Error injection */
 901         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_SET_EVDATA)) {
 902                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 903         } else {
 904                 rrc = gni_ep_set_eventdata(
 905                         ep_hndl, local_event, remote_event);
 906         }
 907
 908         switch (rrc)  {
 909         case GNI_RC_SUCCESS:
 910                 break;
 911         case GNI_RC_INVALID_PARAM:
 912                 GNILND_API_SWBUG(
 913                         ep_hndl, local_event, remote_event);
 914                 break;
 915         default:
 916                 GNILND_API_RC_LBUG(
 917                         ep_hndl, local_event, remote_event);
 918
 919                 /* LBUG never returns, but just for style and consistency */
 920                 break;
 921         }
 922         RETURN(rrc);
 923 }
 924 #undef apick_fn
 925 #undef apick_fmt
 926
 927 #define apick_fn "kgnilnd_ep_unbind"
 928 #define apick_fmt "0x%p"
 929 static inline gni_return_t kgnilnd_ep_unbind(
 930                 IN gni_ep_handle_t      ep_hndl
 931                 )
 932 {
 933         gni_return_t rrc;
 934
 935         /* Error injection */
 936         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_UNBIND)) {
 937                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 938         } else {
 939                 rrc = gni_ep_unbind(
 940                         ep_hndl);
 941         }
 942
 943         switch (rrc)  {
 944         /* both of these are OK, upper SW needs to handle */
 945         case GNI_RC_NOT_DONE:
 946         case GNI_RC_SUCCESS:
 947                 break;
 948         case GNI_RC_INVALID_PARAM:
 949                 GNILND_API_SWBUG(
 950                         ep_hndl);
 951                 break;
 952         default:
 953                 GNILND_API_RC_LBUG(
 954                         ep_hndl);
 955
 956                 /* LBUG never returns, but just for style and consistency */
 957                 break;
 958         }
 959         RETURN(rrc);
 960 }
 961 #undef apick_fn
 962 #undef apick_fmt
 963
 964 #define apick_fn "kgnilnd_ep_destroy"
 965 #define apick_fmt "0x%p"
 966 static inline gni_return_t kgnilnd_ep_destroy(
 967                 IN gni_ep_handle_t      ep_hndl
 968                 )
 969 {
 970         gni_return_t rrc;
 971
 972         /* Error injection */
 973         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_DESTROY)) {
 974                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 975         } else {
 976                 rrc = gni_ep_destroy(
 977                         ep_hndl);
 978         }
 979
 980         switch (rrc)  {
 981         case GNI_RC_SUCCESS:
 982                 break;
 983         case GNI_RC_INVALID_PARAM:
 984                 GNILND_API_SWBUG(
 985                         ep_hndl);
 986                 break;
 987         default:
 988                 GNILND_API_RC_LBUG(
 989                         ep_hndl);
 990
 991                 /* LBUG never returns, but just for style and consistency */
 992                 break;
 993         }
 994         RETURN(rrc);
 995 }
 996 #undef apick_fn
 997 #undef apick_fmt
 998
 999 #define apick_fn "kgnilnd_ep_postdata_w_id"
1000 #define apick_fmt "0x%p, 0x%p, %d, 0x%p, %d, "LPU64""
1001 static inline gni_return_t kgnilnd_ep_postdata_w_id(
1002                 IN gni_ep_handle_t ep_hndl,
1003                 IN void            *in_data,
1004                 IN uint16_t        data_len,
1005                 IN void            *out_buf,
1006                 IN uint16_t        buf_size,
1007                 IN uint64_t        datagram_id
1008                 )
1009 {
1010         gni_return_t rrc;
1011
1012         /* Error injection */
1013         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_POST)) {
1014                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_SIZE_ERROR;
1015         } else {
1016                 rrc = gni_ep_postdata_w_id(
1017                         ep_hndl, in_data, data_len, out_buf, buf_size,
1018                         datagram_id);
1019         }
1020
1021         switch (rrc)  {
1022         case GNI_RC_SUCCESS:
1023         case GNI_RC_ERROR_NOMEM:
1024         case GNI_RC_ERROR_RESOURCE:
1025                 break;
1026         case GNI_RC_INVALID_PARAM:
1027         case GNI_RC_SIZE_ERROR:
1028                 GNILND_API_SWBUG(
1029                         ep_hndl, in_data, data_len, out_buf, buf_size,
1030                         datagram_id);
1031                 break;
1032         default:
1033                 GNILND_API_RC_LBUG(
1034                         ep_hndl, in_data, data_len, out_buf, buf_size,
1035                         datagram_id);
1036
1037                 /* LBUG never returns, but just for style and consistency */
1038                 break;
1039         }
1040         RETURN(rrc);
1041 }
1042 #undef apick_fn
1043 #undef apick_fmt
1044
1045 #define apick_fn "kgnilnd_ep_postdata_test_by_id"
1046 #define apick_fmt "0x%p, "LPU64", 0x%p, 0x%p, 0x%p"
1047 static inline gni_return_t kgnilnd_ep_postdata_test_by_id(
1048                 IN gni_ep_handle_t      ep_hndl,
1049                 IN uint64_t             datagram_id,
1050                 OUT gni_post_state_t    *post_state,
1051                 OUT uint32_t            *remote_addr,
1052                 OUT uint32_t            *remote_id
1053                 )
1054 {
1055         gni_return_t rrc;
1056
1057         /* Error injection */
1058         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_TEST)) {
1059                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_NOMEM;
1060         } else {
1061                 rrc = gni_ep_postdata_test_by_id(
1062                         ep_hndl, datagram_id, post_state, remote_addr,
1063                         remote_id);
1064
1065                 /* we want to lie, but we need to do the actual work first
1066                  * so we don't keep getting the event saying a dgram is ready */
1067                 if (rrc == GNI_RC_SUCCESS && CFS_FAIL_CHECK(CFS_FAIL_GNI_DG_TERMINATE)) {
1068                         /* don't use fail_val, allows us to do FAIL_SOME */
1069                         *post_state = GNI_POST_TERMINATED;
1070                 }
1071         }
1072
1073         switch (rrc)  {
1074         case GNI_RC_SUCCESS:
1075         case GNI_RC_NO_MATCH:
1076                 break;
1077         case GNI_RC_SIZE_ERROR:
1078         case GNI_RC_INVALID_PARAM:
1079                 GNILND_API_SWBUG(
1080                         ep_hndl, datagram_id, post_state, remote_addr,
1081                         remote_id);
1082                 break;
1083         case GNI_RC_ERROR_NOMEM:
1084                 GNILND_API_RESOURCE(
1085                         ep_hndl, datagram_id, post_state, remote_addr,
1086                         remote_id);
1087                 break;
1088         default:
1089                 GNILND_API_RC_LBUG(
1090                         ep_hndl, datagram_id, post_state, remote_addr,
1091                         remote_id);
1092
1093                 /* LBUG never returns, but just for style and consistency */
1094                 break;
1095         }
1096         RETURN(rrc);
1097 }
1098 #undef apick_fn
1099 #undef apick_fmt
1100
1101 #define apick_fn "kgnilnd_ep_postdata_cancel_by_id"
1102 #define apick_fmt "0x%p, "LPU64""
1103 static inline gni_return_t kgnilnd_ep_postdata_cancel_by_id(
1104                 IN gni_ep_handle_t      ep_hndl,
1105                 IN uint64_t             datagram_id
1106                 )
1107 {
1108         gni_return_t rrc;
1109
1110         /* no error injection as the only thing we'd do is LBUG */
1111
1112         rrc = gni_ep_postdata_cancel_by_id(
1113                 ep_hndl, datagram_id);
1114
1115         switch (rrc)  {
1116         case GNI_RC_SUCCESS:
1117         case GNI_RC_NO_MATCH:
1118                 break;
1119         case GNI_RC_INVALID_PARAM:
1120                 GNILND_API_SWBUG(
1121                         ep_hndl, datagram_id);
1122                 break;
1123         default:
1124                 GNILND_API_RC_LBUG(
1125                         ep_hndl, datagram_id);
1126
1127                 /* LBUG never returns, but just for style and consistency */
1128                 break;
1129         }
1130         RETURN(rrc);
1131 }
1132 #undef apick_fn
1133 #undef apick_fmt
1134
1135 #define apick_fn "kgnilnd_postdata_probe_by_id"
1136 #define apick_fmt "0x%p, 0x%p"
1137 static inline gni_return_t kgnilnd_postdata_probe_by_id(
1138                 IN gni_nic_handle_t    nic_hndl,
1139                 OUT uint64_t          *datagram_id
1140                 )
1141 {
1142         gni_return_t rrc;
1143
1144         /* Error injection */
1145         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PROBE)) {
1146                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NO_MATCH;
1147         } else {
1148                 rrc = gni_postdata_probe_by_id(
1149                         nic_hndl, datagram_id);
1150         }
1151
1152         switch (rrc)  {
1153         case GNI_RC_SUCCESS:
1154         case GNI_RC_NO_MATCH:
1155                 break;
1156         case GNI_RC_INVALID_PARAM:
1157                 GNILND_API_SWBUG(
1158                         nic_hndl, datagram_id);
1159                 break;
1160         default:
1161                 GNILND_API_RC_LBUG(
1162                         nic_hndl, datagram_id);
1163
1164                 /* LBUG never returns, but just for style and consistency */
1165                 break;
1166         }
1167         RETURN(rrc);
1168 }
1169 #undef apick_fn
1170 #undef apick_fmt
1171
1172 #define apick_fn "kgnilnd_postdata_probe_wait_by_id"
1173 #define apick_fmt "0x%p, %d, 0x%p"
1174 static inline gni_return_t kgnilnd_postdata_probe_wait_by_id(
1175                 IN gni_nic_handle_t nic_hndl,
1176                 IN uint32_t         timeout,
1177                 OUT uint64_t        *datagram_id
1178                 )
1179 {
1180         gni_return_t rrc;
1181
1182         /* Error injection */
1183         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PROBE_WAIT)) {
1184                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_TIMEOUT;
1185         } else {
1186                 rrc = gni_postdata_probe_wait_by_id(
1187                         nic_hndl, timeout, datagram_id);
1188         }
1189
1190         switch (rrc)  {
1191         case GNI_RC_SUCCESS:
1192         case GNI_RC_TIMEOUT:
1193                 break;
1194         case GNI_RC_INVALID_PARAM:
1195                 GNILND_API_SWBUG(
1196                         nic_hndl, timeout, datagram_id);
1197                 break;
1198         default:
1199                 GNILND_API_RC_LBUG(
1200                         nic_hndl, timeout, datagram_id);
1201
1202                 /* LBUG never returns, but just for style and consistency */
1203                 break;
1204         }
1205         RETURN(rrc);
1206 }
1207 #undef apick_fn
1208 #undef apick_fmt
1209
1210 #define apick_fn "kgnilnd_post_rdma"
1211 #define apick_fmt "0x%p, 0x%p"
1212 static inline gni_return_t kgnilnd_post_rdma(
1213                 IN gni_ep_handle_t               ep_hndl,
1214                 IN gni_post_descriptor_t        *post_descr
1215                 )
1216 {
1217         gni_return_t rrc;
1218
1219         /* Error injection */
1220         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_POST_RDMA)) {
1221                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
1222         } else {
1223                 rrc = gni_post_rdma(
1224                         ep_hndl, post_descr);
1225         }
1226
1227         switch (rrc)  {
1228         case GNI_RC_SUCCESS:
1229                 break;
1230         case GNI_RC_ALIGNMENT_ERROR:
1231         case GNI_RC_INVALID_PARAM:
1232                 GNILND_API_SWBUG(
1233                         ep_hndl, post_descr);
1234                 break;
1235         case GNI_RC_ERROR_RESOURCE:
1236                 CDEBUG(D_NET, "no resources for kgnilnd_post_rdma (0x%p, 0x%p)"
1237                         " rc %s\n", ep_hndl, post_descr,
1238                         kgnilnd_api_rc2str(rrc));
1239                 break;
1240         default:
1241                 GNILND_API_RC_LBUG(
1242                         ep_hndl, post_descr);
1243
1244                 /* LBUG never returns, but just for style and consistency */
1245                 break;
1246         }
1247         RETURN(rrc);
1248 }
1249 #undef apick_fn
1250 #undef apick_fmt
1251
1252 #define apick_fn "kgnilnd_get_completed"
1253 #define apick_fmt "0x%p,"LPX64",0x%p"
1254 static inline gni_return_t kgnilnd_get_completed(
1255                 IN gni_cq_handle_t              cq_hndl,
1256                 IN gni_cq_entry_t               event_data,
1257                 OUT gni_post_descriptor_t       **post_descr
1258                 )
1259 {
1260         gni_return_t rrc;
1261
1262
1263         rrc = gni_get_completed(cq_hndl, event_data, post_descr);
1264
1265         switch (rrc)  {
1266         case GNI_RC_TRANSACTION_ERROR:
1267         case GNI_RC_SUCCESS:
1268                 break;
1269         case GNI_RC_DESCRIPTOR_ERROR:
1270         case GNI_RC_INVALID_PARAM:
1271                 GNILND_API_SWBUG(cq_hndl, event_data, post_descr);
1272                 break;
1273         default:
1274                 GNILND_API_RC_LBUG(cq_hndl, event_data, post_descr);
1275                 /* LBUG never returns, but just for style and consistency */
1276                 break;
1277         }
1278
1279         /* Error injection - we need a valid desc, so let kgni give us one
1280          * - then we lie  */
1281         if (rrc == GNI_RC_SUCCESS &&
1282             (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED))) {
1283                 /* We only trigger TRANSACTION_ERROR for now */
1284                 gni_post_descriptor_t *desc;
1285                 rrc = GNI_RC_TRANSACTION_ERROR;
1286                 desc = *post_descr;
1287                 desc->status = rrc;
1288                 /* recoverable decision made from cfs_fail_val in
1289                  *  kgnilnd_cq_error_str and
1290                  *  kgnilnd_cq_error_recoverable */
1291         }
1292         RETURN(rrc);
1293 }
1294 #undef apick_fn
1295 #undef apick_fmt
1296
1297 #define apick_fn "kgnilnd_cq_error_str"
1298 #define apick_fmt LPX64",0x%p,%d"
1299 static inline gni_return_t kgnilnd_cq_error_str(
1300                 IN gni_cq_entry_t       entry,
1301                 IN void                *buffer,
1302                 IN uint32_t             len
1303                 )
1304 {
1305         gni_return_t rrc;
1306
1307         /* Error injection - set string if we injected a
1308          *  TRANSACTION_ERROR earlier */
1309         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED)) {
1310                 /* if we just set persistent error, we can't ever
1311                  * break in via ssh to clear, so use a count > 10 to indicate fatal */
1312                 sprintf(buffer, "INJECT:%s", cfs_fail_val > 10 ?
1313                         "FATAL" : "RECOVERABLE");
1314                 rrc = GNI_RC_SUCCESS;
1315         } else {
1316                 rrc = gni_cq_error_str(
1317                         entry, buffer, len);
1318         }
1319
1320         switch (rrc)  {
1321         case GNI_RC_SUCCESS:
1322                 break;
1323         case GNI_RC_SIZE_ERROR:
1324         case GNI_RC_INVALID_PARAM:
1325                 GNILND_API_SWBUG(
1326                         entry, buffer, len);
1327                 /* give them something to use */
1328                 snprintf(buffer, len, "UNDEF:UNDEF");
1329                 break;
1330         default:
1331                 GNILND_API_RC_LBUG(
1332                         entry, buffer, len);
1333
1334                 /* LBUG never returns, but just for style and consistency */
1335                 break;
1336         }
1337         RETURN(rrc);
1338 }
1339 #undef apick_fn
1340 #undef apick_fmt
1341
1342 #define apick_fn "kgnilnd_cq_error_recoverable"
1343 #define apick_fmt LPX64",0x%p"
1344 static inline gni_return_t kgnilnd_cq_error_recoverable(
1345                 IN gni_cq_entry_t       entry,
1346                 IN uint32_t            *recoverable
1347                 )
1348 {
1349         gni_return_t rrc;
1350
1351         /* Error injection - set string if we injected a
1352          *  TRANSACTION_ERROR earlier */
1353         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED)) {
1354                 *recoverable = cfs_fail_val > 10 ? 0 : 1;
1355                 rrc = GNI_RC_SUCCESS;
1356         } else {
1357                 rrc = gni_cq_error_recoverable(
1358                         entry, recoverable);
1359         }
1360
1361         switch (rrc)  {
1362         case GNI_RC_SUCCESS:
1363                 break;
1364         case GNI_RC_INVALID_STATE:
1365         case GNI_RC_INVALID_PARAM:
1366                 GNILND_API_SWBUG(
1367                         entry, recoverable);
1368                 *recoverable = 0;
1369                 break;
1370         default:
1371                 GNILND_API_RC_LBUG(
1372                         entry, recoverable);
1373
1374                 /* LBUG never returns, but just for style and consistency */
1375                 break;
1376         }
1377         RETURN(rrc);
1378 }
1379 #undef apick_fn
1380 #undef apick_fmt
1381
1382 #define apick_fn "kgnilnd_mem_register_segments"
1383 #define apick_fmt "0x%p,0x%p,%u,0x%p,%x,0x%p"
1384 static inline gni_return_t
1385 kgnilnd_mem_register_segments(
1386                 IN gni_nic_handle_t     nic_hndl,
1387                 IN gni_mem_segment_t    *mem_segments,
1388                 IN uint32_t             segments_cnt,
1389                 IN gni_cq_handle_t      dst_cq_hndl,
1390                 IN uint32_t             flags,
1391                 OUT gni_mem_handle_t    *mem_hndl
1392                 )
1393 {
1394         gni_return_t rrc;
1395
1396         /* Error injection */
1397         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PHYS_MAP)) {
1398                 rrc = GNI_RC_ERROR_RESOURCE;
1399         } else {
1400                 rrc = gni_mem_register_segments(
1401                         nic_hndl, mem_segments, segments_cnt,
1402                         dst_cq_hndl, flags, mem_hndl);
1403         }
1404
1405         switch (rrc)  {
1406         case GNI_RC_SUCCESS:
1407         case GNI_RC_ERROR_RESOURCE:
1408                 break;
1409         case GNI_RC_INVALID_PARAM:
1410                 GNILND_API_SWBUG(
1411                         nic_hndl, mem_segments, segments_cnt,
1412                         dst_cq_hndl, flags, mem_hndl);
1413                 break;
1414         default:
1415                 GNILND_API_RC_LBUG(
1416                         nic_hndl, mem_segments, segments_cnt,
1417                         dst_cq_hndl, flags, mem_hndl);
1418
1419                 /* LBUG never returns, but just for style and consistency */
1420                 break;
1421         }
1422         RETURN(rrc);
1423 }
1424 #undef apick_fn
1425 #undef apick_fmt
1426
1427 #define apick_fn "kgnilnd_mem_register"
1428 #define apick_fmt "0x%p,"LPX64","LPX64"0x%p,%u,0x%p"
1429 static inline gni_return_t kgnilnd_mem_register(
1430                 IN gni_nic_handle_t     nic_hndl,
1431                 IN uint64_t             address,
1432                 IN uint64_t             length,
1433                 IN gni_cq_handle_t      dst_cq_hndl,
1434                 IN uint32_t             flags,
1435                 OUT gni_mem_handle_t    *mem_hndl
1436                 )
1437 {
1438         gni_return_t rrc;
1439
1440         /* Error injection */
1441         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_MAP)) {
1442                 rrc = GNI_RC_ERROR_RESOURCE;
1443         } else if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_SMALL_MAP) &&
1444                    length <= *kgnilnd_tunables.kgn_max_immediate) {
1445                 rrc = GNI_RC_INVALID_PARAM;
1446         } else {
1447                 rrc = gni_mem_register(
1448                         nic_hndl, address, length,
1449                         dst_cq_hndl, flags, mem_hndl);
1450         }
1451
1452         switch (rrc)  {
1453         case GNI_RC_SUCCESS:
1454         case GNI_RC_ERROR_RESOURCE:
1455                 break;
1456         case GNI_RC_INVALID_PARAM:
1457                 GNILND_API_SWBUG(
1458                         nic_hndl, address, length,
1459                         dst_cq_hndl, flags, mem_hndl);
1460                 break;
1461         default:
1462                 GNILND_API_RC_LBUG(
1463                         nic_hndl, address, length,
1464                         dst_cq_hndl, flags, mem_hndl);
1465
1466                 /* LBUG never returns, but just for style and consistency */
1467                 break;
1468         }
1469         RETURN(rrc);
1470 }
1471 #undef apick_fn
1472 #undef apick_fmt
1473
1474 #define apick_fn "kgnilnd_mem_deregister"
1475 #define apick_fmt "0x%p,0x%p,%d"
1476 static inline gni_return_t kgnilnd_mem_deregister(
1477                 IN gni_nic_handle_t     nic_hndl,
1478                 IN gni_mem_handle_t     *mem_hndl,
1479                 IN int                  hold_timeout
1480                 )
1481 {
1482         gni_return_t rrc;
1483
1484         /* Error injection */
1485         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_UNMAP)) {
1486                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
1487         } else {
1488                 rrc = gni_mem_deregister(
1489                         nic_hndl, mem_hndl, hold_timeout);
1490         }
1491
1492         switch (rrc)  {
1493         case GNI_RC_SUCCESS:
1494                 break;
1495         case GNI_RC_INVALID_PARAM:
1496                 GNILND_API_SWBUG(
1497                         nic_hndl, mem_hndl, hold_timeout);
1498                 break;
1499         default:
1500                 GNILND_API_RC_LBUG(
1501                         nic_hndl, mem_hndl, hold_timeout);
1502
1503                 /* LBUG never returns, but just for style and consistency */
1504                 break;
1505         }
1506         RETURN(rrc);
1507 }
1508 #undef apick_fn
1509 #undef apick_fmt
1510
1511 #define apick_fn "kgnilnd_mem_mdd_release"
1512 #define apick_fmt "0x%p,0x%p"
1513 static inline gni_return_t kgnilnd_mem_mdd_release(
1514                 IN gni_nic_handle_t     nic_hndl,
1515                 IN gni_mem_handle_t     *mem_hndl
1516                 )
1517 {
1518         gni_return_t rrc;
1519
1520         /* Error injection */
1521         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_MDD_RELEASE)) {
1522                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NO_MATCH;
1523         } else {
1524                 rrc = gni_mem_mdd_release(
1525                         nic_hndl, mem_hndl);
1526         }
1527
1528         switch (rrc)  {
1529         case GNI_RC_SUCCESS:
1530         case GNI_RC_NO_MATCH:
1531                 break;
1532         default:
1533                 GNILND_API_RC_LBUG(
1534                         nic_hndl, mem_hndl);
1535
1536                 /* LBUG never returns, but just for style and consistency */
1537                 break;
1538         }
1539         RETURN(rrc);
1540 }
1541 #undef apick_fn
1542 #undef apick_fmt
1543
1544 #endif /* _GNILND_API_WRAP_H */