lnet/klnds/gnilnd/gnilnd_api_wrap.h

   1 /*
   2  * Copyright (C) 2009-2012 Cray, Inc.
   3  *
   4  *   Author: Nic Henke <nic@cray.com>
   5  *
   6  *   This file is part of Lustre, http://www.lustre.org.
   7  *
   8  *   Lustre is free software; you can redistribute it and/or
   9  *   modify it under the terms of version 2 of the GNU General Public
  10  *   License as published by the Free Software Foundation.
  11  *
  12  *   Lustre is distributed in the hope that it will be useful,
  13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *   GNU General Public License for more details.
  16  *
  17  *   You should have received a copy of the GNU General Public License
  18  *   along with Lustre; if not, write to the Free Software
  19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20  *
  21  */
  22 #ifndef _GNILND_API_WRAP_H
  23 #define _GNILND_API_WRAP_H
  24
  25 /* LNet is allocated failure locations 0xe000 to 0xffff */
  26
  27 /* GNILND has 0xf0XX */
  28 #define CFS_FAIL_GNI                    0xf000
  29 #define CFS_FAIL_GNI_PHYS_MAP           0xf001
  30 #define CFS_FAIL_GNI_VIRT_MAP           0xf002
  31 #define CFS_FAIL_GNI_GET_UNMAP          0xf003
  32 #define CFS_FAIL_GNI_PUT_UNMAP          0xf004
  33 #define CFS_FAIL_GNI_MAP_TX             0xf005
  34 #define CFS_FAIL_GNI_SMSG_SEND          0xf006
  35 #define CFS_FAIL_GNI_CLOSE_SEND         0xf007
  36 #define CFS_FAIL_GNI_CDM_CREATE         0xf008
  37 #define CFS_FAIL_GNI_CDM_DESTROY        0xf009
  38 #define CFS_FAIL_GNI_CDM_ATTACH         0xf00a
  39 #define CFS_FAIL_GNI_CQ_CREATE          0xf00b
  40 #define CFS_FAIL_GNI_CQ_DESTROY         0xf00c
  41 #define CFS_FAIL_GNI_EP_BIND            0xf00d
  42 #define CFS_FAIL_GNI_EP_UNBIND          0xf00e
  43 #define CFS_FAIL_GNI_EP_SET_EVDATA      0xf00f
  44 #define CFS_FAIL_GNI_SMSG_INIT          0xf010
  45 #define CFS_FAIL_GNI_SMSG_RELEASE       0xf011
  46 #define CFS_FAIL_GNI_POST_RDMA          0xf012
  47 #define CFS_FAIL_GNI_GET_COMPLETED      0xf013
  48 #define CFS_FAIL_GNI_EP_DESTROY         0xf015
  49 #define CFS_FAIL_GNI_VIRT_UNMAP         0xf016
  50 #define CFS_FAIL_GNI_MDD_RELEASE        0xf017
  51 #define CFS_FAIL_GNI_NOOP_SEND          0xf018
  52 #define CFS_FAIL_GNI_ERR_SUBSCRIBE      0xf01a
  53 #define CFS_FAIL_GNI_QUIESCE_RACE       0xf01b
  54 #define CFS_FAIL_GNI_DG_TERMINATE       0xf01c
  55 #define CFS_FAIL_GNI_REG_QUIESCE        0xf01d
  56 #define CFS_FAIL_GNI_IN_QUIESCE         0xf01e
  57 #define CFS_FAIL_GNI_DELAY_RDMA         0xf01f
  58 #define CFS_FAIL_GNI_SR_DOWN_RACE       0xf020
  59 #define CFS_FAIL_GNI_ALLOC_TX           0xf021
  60 #define CFS_FAIL_GNI_FMABLK_AVAIL       0xf022
  61 #define CFS_FAIL_GNI_EP_CREATE          0xf023
  62 #define CFS_FAIL_GNI_CQ_GET_EVENT       0xf024
  63 #define CFS_FAIL_GNI_PROBE              0xf025
  64 #define CFS_FAIL_GNI_EP_TEST            0xf026
  65 #define CFS_FAIL_GNI_CONNREQ_DROP       0xf027
  66 #define CFS_FAIL_GNI_CONNREQ_PROTO      0xf028
  67 #define CFS_FAIL_GNI_CONND_PILEUP       0xf029
  68 #define CFS_FAIL_GNI_PHYS_SETUP         0xf02a
  69 #define CFS_FAIL_GNI_FIND_TARGET        0xf02b
  70 #define CFS_FAIL_GNI_WC_DGRAM_FREE      0xf02c
  71 #define CFS_FAIL_GNI_DROP_CLOSING       0xf02d
  72 #define CFS_FAIL_GNI_RX_CLOSE_CLOSING   0xf02e
  73 #define CFS_FAIL_GNI_RX_CLOSE_CLOSED    0xf02f
  74 #define CFS_FAIL_GNI_EP_POST            0xf030
  75 #define CFS_FAIL_GNI_PACK_SRCNID        0xf031
  76 #define CFS_FAIL_GNI_PACK_DSTNID        0xf032
  77 #define CFS_FAIL_GNI_PROBE_WAIT         0xf033
  78 #define CFS_FAIL_GNI_SMSG_CKSUM1        0xf034
  79 #define CFS_FAIL_GNI_SMSG_CKSUM2        0xf035
  80 #define CFS_FAIL_GNI_SMSG_CKSUM3        0xf036
  81 #define CFS_FAIL_GNI_DROP_DESTROY_EP    0xf037
  82 #define CFS_FAIL_GNI_SMSG_GETNEXT       0xf038
  83 #define CFS_FAIL_GNI_FINISH_PURG        0xf039
  84 #define CFS_FAIL_GNI_PURG_REL_DELAY     0xf03a
  85 #define CFS_FAIL_GNI_DONT_NOTIFY        0xf03b
  86 #define CFS_FAIL_GNI_VIRT_SMALL_MAP     0xf03c
  87 #define CFS_FAIL_GNI_DELAY_RDMAQ        0xf03d
  88 #define CFS_FAIL_GNI_PAUSE_SHUTDOWN     0xf03e
  89 #define CFS_FAIL_GNI_PAUSE_DGRAM_COMP   0xf03f
  90 #define CFS_FAIL_GNI_NET_LOOKUP         0xf040
  91 #define CFS_FAIL_GNI_RECV_TIMEOUT       0xf041
  92 #define CFS_FAIL_GNI_SEND_TIMEOUT       0xf042
  93 #define CFS_FAIL_GNI_ONLY_NOOP          0xf043
  94 #define CFS_FAIL_GNI_FINISH_PURG2       0xf044
  95 #define CFS_FAIL_GNI_RACE_RESET         0xf045
  96 #define CFS_FAIL_GNI_GNP_CONNECTING1    0xf046
  97 #define CFS_FAIL_GNI_GNP_CONNECTING2    0xf047
  98 #define CFS_FAIL_GNI_GNP_CONNECTING3    0xf048
  99 #define CFS_FAIL_GNI_SCHEDULE_COMPLETE  0xf049
 100 #define CFS_FAIL_GNI_PUT_ACK_AGAIN      0xf050
 101 #define CFS_FAIL_GNI_GET_REQ_AGAIN      0xf051
 102 #define CFS_FAIL_GNI_SCHED_DEADLINE     0xf052
 103 #define CFS_FAIL_GNI_DGRAM_DEADLINE     0xf053
 104 #define CFS_FAIL_GNI_DGRAM_DROP_TX      0xf054
 105 #define CFS_FAIL_GNI_RDMA_CQ_ERROR      0xf055
 106
 107 /* helper macros */
 108 extern void
 109 _kgnilnd_api_rc_lbug(const char *rcstr, int rc, struct libcfs_debug_msg_data *data,
 110                         const char *fmt, ...)
 111         __attribute__ ((format (printf, 4, 5)));
 112
 113 #define kgnilnd_api_rc_lbug(msgdata, rc, fmt, a...)                             \
 114 do {                                                                            \
 115         CFS_CHECK_STACK(msgdata, D_ERROR, NULL);                                \
 116         /* we don't mask this - it is always at D_ERROR */                      \
 117         _kgnilnd_api_rc_lbug(kgnilnd_api_rc2str(rc), (rc), msgdata, fmt, ##a);  \
 118 } while (0)
 119
 120 #define DO_RETCODE(x) case x: return #x;
 121 static inline const char *
 122 kgnilnd_api_rc2str(gni_return_t rrc)
 123 {
 124
 125         switch (rrc) {
 126                 DO_RETCODE(GNI_RC_SUCCESS)
 127                 DO_RETCODE(GNI_RC_NOT_DONE);
 128                 DO_RETCODE(GNI_RC_INVALID_PARAM);
 129                 DO_RETCODE(GNI_RC_ERROR_RESOURCE);
 130                 DO_RETCODE(GNI_RC_TIMEOUT);
 131                 DO_RETCODE(GNI_RC_PERMISSION_ERROR);
 132                 DO_RETCODE(GNI_RC_DESCRIPTOR_ERROR);
 133                 DO_RETCODE(GNI_RC_ALIGNMENT_ERROR);
 134                 DO_RETCODE(GNI_RC_INVALID_STATE);
 135                 DO_RETCODE(GNI_RC_NO_MATCH);
 136                 DO_RETCODE(GNI_RC_SIZE_ERROR);
 137                 DO_RETCODE(GNI_RC_TRANSACTION_ERROR);
 138                 DO_RETCODE(GNI_RC_ILLEGAL_OP);
 139                 DO_RETCODE(GNI_RC_ERROR_NOMEM);
 140         }
 141         LBUG();
 142 }
 143 #undef DO_RETCODE
 144
 145 /* log an error and LBUG for unhandled rc from gni api function
 146  * the fmt should be something like:
 147  *  gni_api_call(arg1, arg2, arg3)
 148  */
 149
 150 /* apick_fn and apick_fmt should be defined for each site */
 151 #undef apick_fn
 152 #undef apick_fmt
 153
 154 #define GNILND_API_RC_LBUG(args...)                                             \
 155 do {                                                                            \
 156         LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);                     \
 157         kgnilnd_api_rc_lbug(&msgdata, rrc, apick_fn"("apick_fmt")", ##args);    \
 158 } while (0)
 159
 160 #define GNILND_API_SWBUG(args...)                                               \
 161 do {                                                                            \
 162         CERROR("likely SOFTWARE BUG "apick_fn"("apick_fmt") rc %s\n",           \
 163                  ##args, kgnilnd_api_rc2str(rrc));                              \
 164 } while (0)
 165
 166 #define GNILND_API_EINVAL(args...)                                              \
 167 do {                                                                            \
 168         CERROR("invalid parameter to "apick_fn"("apick_fmt") rc %s\n",          \
 169                  ##args, kgnilnd_api_rc2str(rrc));                              \
 170 } while (0)
 171
 172 #define GNILND_API_RESOURCE(args...)                                            \
 173 do {                                                                            \
 174         CERROR("no resources for "apick_fn"("apick_fmt") rc %s\n",              \
 175                 ##args, kgnilnd_api_rc2str(rrc));                               \
 176 } while (0)
 177
 178 #define GNILND_API_BUSY(args...)                                                \
 179 do {                                                                            \
 180         CERROR("resources busy for "apick_fn"("apick_fmt") rc %s\n",            \
 181                 ##args, kgnilnd_api_rc2str(rrc));                               \
 182 } while (0)
 183
 184 #undef DEBUG_SMSG_CREDITS
 185 #ifdef DEBUG_SMSG_CREDITS
 186 #define CRAY_CONFIG_GHAL_GEMINI
 187 #include <gni_priv.h>
 188 #define GNIDBG_SMSG_CREDS(level, conn)                                        \
 189 do {                                                                          \
 190         gni_ep_smsg_mbox_t *smsg = conn->gnc_ephandle->smsg;                  \
 191         CDEBUG(level, "SMSGDBG: conn %p mcred %d/%d bcred %d/%d "             \
 192                 "s_seq %d/%d/%d r_seq %d/%d/%d retr %d\n",                    \
 193                 conn, smsg->mbox_credits, smsg->back_mbox_credits,            \
 194                 smsg->buffer_credits, smsg->back_buffer_credits,              \
 195                 smsg->s_seqno, smsg->s_seqno_back_mbox_credits,               \
 196                 smsg->s_seqno_back_buffer_credits, smsg->r_seqno,             \
 197                 smsg->r_seqno_back_mbox_credits,                              \
 198                 smsg->r_seqno_back_buffer_credits, smsg->retransmit_count);   \
 199 } while (0)
 200 #else
 201 #define GNIDBG_SMSG_CREDS(level, conn) do {} while(0)
 202 #endif
 203
 204 /* these are all wrappers around gni_XXX functions.
 205  * This allows us to handle all the return codes and api checks without
 206  * dirtying up the logic code */
 207
 208 /* TODO: RETURN wrapper that translates integer to GNI API RC string */
 209
 210 #define apick_fn "kgnilnd_cdm_create"
 211 #define apick_fmt "%u, %u, %u, %u, 0x%p"
 212 static inline gni_return_t kgnilnd_cdm_create(
 213                 IN uint32_t             inst_id,
 214                 IN uint8_t              ptag,
 215                 IN uint32_t             cookie,
 216                 IN uint32_t             modes,
 217                 OUT gni_cdm_handle_t    *cdm_hndl
 218                 )
 219 {
 220         gni_return_t rrc;
 221
 222         /* Error injection */
 223         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CDM_CREATE)) {
 224                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 225         } else {
 226                 rrc = gni_cdm_create(inst_id, ptag, cookie, modes, cdm_hndl);
 227         }
 228
 229         switch (rrc)  {
 230         case GNI_RC_SUCCESS:
 231                 break;
 232         case GNI_RC_ERROR_RESOURCE:
 233         case GNI_RC_INVALID_PARAM:
 234                 /* Try to bail gracefully */
 235                 GNILND_API_SWBUG(
 236                         inst_id, ptag, cookie, modes, cdm_hndl);
 237                 break;
 238         default:
 239                 GNILND_API_RC_LBUG(
 240                         inst_id, ptag, cookie, modes, cdm_hndl);
 241
 242                 /* LBUG never returns, but just for style and consistency */
 243                 break;
 244         }
 245         RETURN(rrc);
 246 }
 247
 248 #undef apick_fn
 249 #undef apick_fmt
 250
 251 #define apick_fn "kgnilnd_cdm_attach"
 252 #define apick_fmt "0x%p, %u, 0x%p, 0x%p"
 253 static inline gni_return_t kgnilnd_cdm_attach(
 254                 IN gni_cdm_handle_t     cdm_hndl,
 255                 IN uint32_t             device_id,
 256                 OUT uint32_t            *local_addr,
 257                 OUT gni_nic_handle_t    *nic_hndl
 258                 )
 259 {
 260         gni_return_t rrc;
 261
 262         /* Error injection */
 263         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CDM_ATTACH)) {
 264                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 265         } else {
 266                 rrc = gni_cdm_attach(cdm_hndl, device_id, local_addr, nic_hndl);
 267         }
 268
 269         switch (rrc)  {
 270         case GNI_RC_SUCCESS:
 271                 break;
 272         case GNI_RC_NO_MATCH:
 273         case GNI_RC_INVALID_PARAM:
 274                 GNILND_API_SWBUG(
 275                         cdm_hndl, device_id, local_addr, nic_hndl);
 276                 break;
 277         case GNI_RC_ERROR_RESOURCE:
 278         case GNI_RC_INVALID_STATE:
 279                 GNILND_API_RESOURCE(
 280                         cdm_hndl, device_id, local_addr, nic_hndl);
 281                 break;
 282         default:
 283                 GNILND_API_RC_LBUG(
 284                         cdm_hndl, device_id, local_addr, nic_hndl);
 285
 286                 /* LBUG never returns, but just for style and consistency */
 287                 break;
 288         }
 289         RETURN(rrc);
 290 }
 291 #undef apick_fmt
 292 #undef apick_fn
 293
 294 #define apick_fn "kgnilnd_cdm_destroy"
 295 #define apick_fmt "0x%p"
 296 static inline gni_return_t kgnilnd_cdm_destroy(
 297                 IN gni_cdm_handle_t     cdm_hndl
 298                 )
 299 {
 300         gni_return_t rrc;
 301
 302         /* Error injection */
 303         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_DESTROY)) {
 304                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 305         } else {
 306                 rrc = gni_cdm_destroy(
 307                         cdm_hndl);
 308         }
 309
 310         switch (rrc)  {
 311         case GNI_RC_SUCCESS:
 312                 break;
 313         case GNI_RC_INVALID_PARAM:
 314                 GNILND_API_SWBUG(
 315                         cdm_hndl);
 316                 break;
 317         default:
 318                 GNILND_API_RC_LBUG(
 319                         cdm_hndl);
 320
 321                 /* LBUG never returns, but just for style and consistency */
 322                 break;
 323         }
 324         RETURN(rrc);
 325 }
 326 #undef apick_fn
 327 #undef apick_fmt
 328
 329 #define apick_fn "kgnilnd_subscribe_errors"
 330 #define apick_fmt "0x%p,%x,%u,0x%p,0x%p,0x%p"
 331 static inline gni_return_t kgnilnd_subscribe_errors(
 332                 IN gni_nic_handle_t  nic_handle,
 333                 IN gni_error_mask_t  mask,
 334                 IN uint32_t          EEQ_size,
 335                 IN void              (*EQ_new_event)(gni_err_handle_t),
 336                 IN void              (*app_crit_err)(gni_err_handle_t),
 337                 OUT gni_err_handle_t *err_handle
 338                 )
 339 {
 340         gni_return_t rrc;
 341
 342         /* Error injection */
 343         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_ERR_SUBSCRIBE)) {
 344                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 345         } else {
 346                 rrc = gni_subscribe_errors(
 347                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 348                         err_handle);
 349         }
 350
 351         switch (rrc)  {
 352         case GNI_RC_SUCCESS:
 353                 break;
 354         case GNI_RC_INVALID_PARAM:
 355                 GNILND_API_SWBUG(
 356                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 357                         err_handle);
 358                 break;
 359         case GNI_RC_ERROR_RESOURCE:
 360                 GNILND_API_RESOURCE(
 361                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 362                         err_handle);
 363                 break;
 364         default:
 365                 GNILND_API_RC_LBUG(
 366                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 367                         err_handle);
 368                 /* LBUG never returns, but just for style and consistency */
 369                 break;
 370         }
 371         RETURN(rrc);
 372 }
 373 #undef apick_fn
 374 #undef apick_fmt
 375
 376 #define apick_fn "kgnilnd_release_errors"
 377 #define apick_fmt "0x%p"
 378 static inline gni_return_t kgnilnd_release_errors(
 379                 IN gni_err_handle_t err_handle
 380                 )
 381 {
 382         gni_return_t rrc;
 383
 384         rrc = gni_release_errors(
 385                         err_handle);
 386
 387         switch (rrc)  {
 388         case GNI_RC_SUCCESS:
 389                 break;
 390         case GNI_RC_INVALID_PARAM:
 391         case GNI_RC_NOT_DONE:
 392                 GNILND_API_SWBUG(
 393                         err_handle);
 394                 break;
 395         default:
 396                 GNILND_API_RC_LBUG(
 397                         err_handle);
 398                 /* LBUG never returns, but just for style and consistency */
 399                 break;
 400         }
 401         RETURN(rrc);
 402 }
 403 #undef apick_fn
 404 #undef apick_fmt
 405
 406 #define apick_fn "kgnilnd_set_quiesce_callback"
 407 #define apick_fmt "0x%p,0x%p"
 408 static inline gni_return_t kgnilnd_set_quiesce_callback(
 409                 IN gni_nic_handle_t  nic_handle,
 410                 IN void              (*qsce_func)(gni_nic_handle_t, uint64_t msecs)
 411                 )
 412 {
 413         gni_return_t rrc;
 414
 415         /* Error injection */
 416         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_REG_QUIESCE)) {
 417                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 418         } else {
 419                 rrc = gni_set_quiesce_callback(
 420                         nic_handle, qsce_func);
 421         }
 422
 423         switch (rrc)  {
 424         case GNI_RC_SUCCESS:
 425                 break;
 426         case GNI_RC_INVALID_STATE:
 427         case GNI_RC_INVALID_PARAM:
 428                 GNILND_API_SWBUG(
 429                         nic_handle, qsce_func);
 430                 break;
 431         default:
 432                 GNILND_API_RC_LBUG(
 433                         nic_handle, qsce_func);
 434                 /* LBUG never returns, but just for style and consistency */
 435                 break;
 436         }
 437         RETURN(rrc);
 438 }
 439 #undef apick_fn
 440 #undef apick_fmt
 441
 442 #define apick_fn "kgnilnd_get_quiesce_status"
 443 #define apick_fmt "0x%p"
 444 static inline gni_return_t kgnilnd_get_quiesce_status(
 445                 IN gni_nic_handle_t  nic_handle
 446                 )
 447 {
 448         uint32_t rrc;
 449
 450         /* this has weird RC -
 451          * 0 - quiesce not in progress
 452          * 1 - quiesce is turned on
 453         */
 454
 455         /* Error injection */
 456         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_IN_QUIESCE)) {
 457                 rrc = 1;
 458         } else {
 459                 rrc = gni_get_quiesce_status(
 460                         nic_handle);
 461         }
 462
 463         switch (rrc)  {
 464         case 1:
 465         case 0:
 466                 break;
 467         default:
 468                 GNILND_API_RC_LBUG(
 469                         nic_handle);
 470                 /* LBUG never returns, but just for style and consistency */
 471                 break;
 472         }
 473         RETURN(rrc);
 474 }
 475 #undef apick_fn
 476 #undef apick_fmt
 477
 478 #define apick_fn "kgnilnd_cq_create"
 479 #define apick_fmt "0x%p, %u, %u, 0x%p, %#llx, 0x%p"
 480 static inline gni_return_t kgnilnd_cq_create(
 481                 IN gni_nic_handle_t     nic_hndl,
 482                 IN uint32_t             entry_count,
 483                 IN uint32_t             delay_index,
 484                 IN gni_cq_event_hndlr_f *event_handler,
 485                 IN uint64_t             usr_event_data,
 486                 OUT gni_cq_handle_t     *cq_hndl
 487                 )
 488 {
 489         gni_return_t rrc;
 490
 491         /* Error injection */
 492         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_CREATE)) {
 493                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 494         } else {
 495                 rrc = gni_cq_create(
 496                        nic_hndl, entry_count, delay_index, event_handler,
 497                         usr_event_data, cq_hndl);
 498         }
 499
 500         switch (rrc)  {
 501         case GNI_RC_SUCCESS:
 502                 break;
 503         case GNI_RC_INVALID_PARAM:
 504                 GNILND_API_SWBUG(
 505                         nic_hndl, entry_count, delay_index, event_handler,
 506                         usr_event_data, cq_hndl);
 507                 break;
 508         case GNI_RC_ERROR_RESOURCE:
 509                 GNILND_API_RESOURCE(
 510                         nic_hndl, entry_count, delay_index, event_handler,
 511                         usr_event_data, cq_hndl);
 512                 break;
 513         default:
 514                 GNILND_API_RC_LBUG(
 515                         nic_hndl, entry_count, delay_index, event_handler,
 516                         usr_event_data, cq_hndl);
 517
 518                 /* LBUG never returns, but just for style and consistency */
 519                 break;
 520         }
 521         RETURN(rrc);
 522 }
 523 #undef apick_fn
 524 #undef apick_fmt
 525
 526 #define apick_fn "kgnilnd_cq_destroy"
 527 #define apick_fmt "0x%p"
 528 static inline gni_return_t kgnilnd_cq_destroy(
 529                 IN gni_cq_handle_t cq_hndl
 530                 )
 531 {
 532         gni_return_t rrc;
 533
 534         /* Error injection */
 535         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_DESTROY)) {
 536                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 537         } else {
 538
 539                 rrc = gni_cq_destroy(
 540                         cq_hndl);
 541         }
 542
 543         switch (rrc)  {
 544         case GNI_RC_SUCCESS:
 545                 break;
 546         case GNI_RC_INVALID_PARAM:
 547                 GNILND_API_SWBUG(
 548                         cq_hndl);
 549                 break;
 550         case GNI_RC_ERROR_RESOURCE:
 551                 GNILND_API_BUSY(
 552                         cq_hndl);
 553                 break;
 554         default:
 555                 GNILND_API_RC_LBUG(
 556                         cq_hndl);
 557
 558                 /* LBUG never returns, but just for style and consistency */
 559                 break;
 560         }
 561         RETURN(rrc);
 562 }
 563 #undef apick_fn
 564 #undef apick_fmt
 565
 566 #define apick_fn "kgnilnd_cq_get_event"
 567 #define apick_fmt "0x%p, 0x%p"
 568 static inline gni_return_t kgnilnd_cq_get_event(
 569                 IN gni_cq_handle_t cq_hndl,
 570                 OUT gni_cq_entry_t *event_data
 571                 )
 572 {
 573         gni_return_t rrc;
 574
 575         /* no error injection - CQs are touchy about the data.
 576          * where appropriate, we'll do this on the CQs that should be able to
 577          * handle the various errors */
 578         rrc = gni_cq_get_event(
 579                         cq_hndl, event_data);
 580
 581         switch (rrc)  {
 582         case GNI_RC_SUCCESS:
 583         case GNI_RC_NOT_DONE:
 584         case GNI_RC_TRANSACTION_ERROR:
 585                 break;
 586         case GNI_RC_ERROR_RESOURCE:
 587                 LASSERTF(GNI_CQ_OVERRUN(*event_data),
 588                          "kgni returned ERROR_RESOURCE but cq_hndl 0x%p is not "
 589                          "overrun\n", cq_hndl);
 590                 break;
 591         case GNI_RC_INVALID_PARAM:
 592                 GNILND_API_SWBUG(
 593                         cq_hndl, event_data);
 594                 break;
 595         default:
 596                 GNILND_API_RC_LBUG(
 597                         cq_hndl, event_data);
 598
 599                 /* LBUG never returns, but just for style and consistency */
 600                 break;
 601         }
 602         return rrc;
 603 }
 604 #undef apick_fn
 605 #undef apick_fmt
 606
 607 #define apick_fn "kgnilnd_smsg_init"
 608 #define apick_fmt "0x%p, 0x%p, 0x%p"
 609 static inline gni_return_t kgnilnd_smsg_init(
 610                 IN gni_ep_handle_t      ep_hndl,
 611                 IN gni_smsg_attr_t      *local_smsg_attr,
 612                 IN gni_smsg_attr_t      *remote_smsg_attr
 613                 )
 614 {
 615         gni_return_t rrc;
 616
 617         /* Error injection */
 618         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_INIT)) {
 619                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_RESOURCE;
 620         } else {
 621                 rrc = gni_smsg_init(
 622                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 623         }
 624
 625         switch (rrc)  {
 626         /* both of these are OK, upper SW needs to handle */
 627         case GNI_RC_SUCCESS:
 628         case GNI_RC_NOT_DONE:
 629                 break;
 630         case GNI_RC_INVALID_PARAM:
 631         case GNI_RC_INVALID_STATE:
 632                 GNILND_API_SWBUG(
 633                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 634                 break;
 635         case GNI_RC_ERROR_RESOURCE:
 636                 GNILND_API_RESOURCE(
 637                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 638                 break;
 639         default:
 640                 GNILND_API_RC_LBUG(
 641                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 642
 643                 /* LBUG never returns, but just for style and consistency */
 644                 break;
 645         }
 646         RETURN(rrc);
 647 }
 648 #undef apick_fn
 649 #undef apick_fmt
 650
 651 #define apick_fn "kgnilnd_smsg_send"
 652 #define apick_fmt "0x%p, 0x%p, %d, 0x%p, %u %u"
 653 static inline gni_return_t kgnilnd_smsg_send(
 654                 IN gni_ep_handle_t      ep_hndl,
 655                 IN void                 *header,
 656                 IN uint32_t             header_length,
 657                 IN void                 *data,
 658                 IN uint32_t             data_length,
 659                 IN uint32_t             msg_id
 660                 )
 661 {
 662         gni_return_t rrc;
 663
 664         /* Error injection */
 665         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_SEND)) {
 666                 if (cfs_fail_loc & CFS_FAIL_RAND) {
 667                         rrc = GNI_RC_NOT_DONE;
 668                 } else {
 669                         rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 670                 }
 671         } else {
 672                 rrc = gni_smsg_send(
 673                         ep_hndl, header, header_length, data, data_length, msg_id);
 674         }
 675
 676         switch (rrc)  {
 677         /* both of these are OK, upper SW needs to handle */
 678         case GNI_RC_SUCCESS:
 679         case GNI_RC_NOT_DONE:
 680                 break;
 681         case GNI_RC_INVALID_PARAM:
 682                 GNILND_API_SWBUG(
 683                         ep_hndl, header, header_length, data, data_length, msg_id);
 684                 break;
 685         case GNI_RC_ERROR_RESOURCE:
 686                 GNILND_API_RESOURCE(
 687                         ep_hndl, header, header_length, data, data_length, msg_id);
 688                 break;
 689         default:
 690                 GNILND_API_RC_LBUG(
 691                         ep_hndl, header, header_length, data, data_length, msg_id);
 692
 693                 /* LBUG never returns, but just for style and consistency */
 694                 break;
 695         }
 696         RETURN(rrc);
 697 }
 698 #undef apick_fn
 699 #undef apick_fmt
 700
 701 #define apick_fn "kgnilnd_smsg_getnext"
 702 #define apick_fmt "0x%p,0x%p"
 703 static inline gni_return_t kgnilnd_smsg_getnext(
 704                 IN gni_ep_handle_t      ep_hndl,
 705                 OUT void                **header
 706                 )
 707 {
 708         gni_return_t rrc;
 709
 710         /* Error injection */
 711         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_RELEASE)) {
 712                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 713         } else {
 714                 rrc = gni_smsg_getnext(
 715                         ep_hndl, header);
 716         }
 717
 718         switch (rrc)  {
 719         /* both of these are OK, upper SW needs to handle */
 720         case GNI_RC_SUCCESS:
 721         case GNI_RC_NOT_DONE:
 722         case GNI_RC_INVALID_STATE:
 723                 break;
 724         case GNI_RC_INVALID_PARAM:
 725                 GNILND_API_SWBUG(
 726                         ep_hndl, header);
 727                 break;
 728         default:
 729                 GNILND_API_RC_LBUG(
 730                         ep_hndl, header);
 731
 732                 /* LBUG never returns, but just for style and consistency */
 733                 break;
 734         }
 735         RETURN(rrc);
 736 }
 737 #undef apick_fn
 738 #undef apick_fmt
 739
 740 #define apick_fn "kgnilnd_smsg_release"
 741 #define apick_fmt "0x%p"
 742 static inline gni_return_t kgnilnd_smsg_release(
 743                 IN gni_ep_handle_t      ep_hndl
 744                 )
 745 {
 746         gni_return_t rrc;
 747
 748         /* Error injection */
 749         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_RELEASE)) {
 750                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 751         } else {
 752                 rrc = gni_smsg_release(
 753                         ep_hndl);
 754         }
 755
 756         switch (rrc)  {
 757         /* both of these are OK, upper SW needs to handle */
 758         case GNI_RC_SUCCESS:
 759         case GNI_RC_NOT_DONE:
 760                 break;
 761         case GNI_RC_INVALID_PARAM:
 762                 GNILND_API_SWBUG(
 763                         ep_hndl);
 764                 break;
 765         default:
 766                 GNILND_API_RC_LBUG(
 767                         ep_hndl);
 768
 769                 /* LBUG never returns, but just for style and consistency */
 770                 break;
 771         }
 772         RETURN(rrc);
 773 }
 774 #undef apick_fn
 775 #undef apick_fmt
 776
 777 #define apick_fn "kgnilnd_ep_create"
 778 #define apick_fmt "0x%p, 0x%p, 0x%p"
 779 static inline gni_return_t kgnilnd_ep_create(
 780                 IN gni_nic_handle_t     nic_hndl,
 781                 IN gni_cq_handle_t      src_cq_hndl,
 782                 OUT gni_ep_handle_t     *ep_hndl
 783                 )
 784 {
 785         gni_return_t rrc;
 786
 787         /* error injection */
 788         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_CREATE)) {
 789                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_NOMEM;
 790         } else {
 791                 rrc = gni_ep_create(
 792                         nic_hndl, src_cq_hndl, ep_hndl);
 793         }
 794
 795         switch (rrc)  {
 796         case GNI_RC_SUCCESS:
 797                 break;
 798         case GNI_RC_INVALID_PARAM:
 799                 GNILND_API_SWBUG(
 800                         nic_hndl, src_cq_hndl, ep_hndl);
 801                 break;
 802         case GNI_RC_ERROR_NOMEM:
 803                 GNILND_API_RESOURCE(
 804                         nic_hndl, src_cq_hndl, ep_hndl);
 805                 break;
 806         default:
 807                 GNILND_API_RC_LBUG(
 808                         nic_hndl, src_cq_hndl, ep_hndl);
 809
 810                 /* lbug never returns, but just for style and consistency */
 811                 break;
 812         }
 813         RETURN(rrc);
 814 }
 815 #undef apick_fn
 816 #undef apick_fmt
 817
 818 #define apick_fn "kgnilnd_ep_bind"
 819 #define apick_fmt "0x%p, %x, %x"
 820 static inline gni_return_t kgnilnd_ep_bind(
 821                 IN gni_ep_handle_t      ep_hndl,
 822                 IN uint32_t             remote_addr,
 823                 IN uint32_t             remote_id
 824                 )
 825 {
 826         gni_return_t rrc;
 827
 828         /* error injection */
 829         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_BIND)) {
 830                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 831         } else {
 832                 rrc = gni_ep_bind(
 833                         ep_hndl, remote_addr, remote_id);
 834         }
 835
 836         switch (rrc)  {
 837         /* both of these are ok, upper sw needs to handle */
 838         case GNI_RC_SUCCESS:
 839         case GNI_RC_NOT_DONE:
 840                 break;
 841         case GNI_RC_INVALID_PARAM:
 842                 GNILND_API_SWBUG(
 843                         ep_hndl, remote_addr, remote_id);
 844                 break;
 845         default:
 846                 GNILND_API_RC_LBUG(
 847                         ep_hndl, remote_addr, remote_id);
 848
 849                 /* lbug never returns, but just for style and consistency */
 850                 break;
 851         }
 852         RETURN(rrc);
 853 }
 854 #undef apick_fn
 855 #undef apick_fmt
 856
 857 #define apick_fn "kgnilnd_ep_set_eventdata"
 858 #define apick_fmt "0x%p, %x, %x"
 859 static inline gni_return_t kgnilnd_ep_set_eventdata(
 860                 IN gni_ep_handle_t      ep_hndl,
 861                 IN uint32_t             local_event,
 862                 IN uint32_t             remote_event
 863                 )
 864 {
 865         gni_return_t rrc;
 866
 867         /* Error injection */
 868         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_SET_EVDATA)) {
 869                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 870         } else {
 871                 rrc = gni_ep_set_eventdata(
 872                         ep_hndl, local_event, remote_event);
 873         }
 874
 875         switch (rrc)  {
 876         case GNI_RC_SUCCESS:
 877                 break;
 878         case GNI_RC_INVALID_PARAM:
 879                 GNILND_API_SWBUG(
 880                         ep_hndl, local_event, remote_event);
 881                 break;
 882         default:
 883                 GNILND_API_RC_LBUG(
 884                         ep_hndl, local_event, remote_event);
 885
 886                 /* LBUG never returns, but just for style and consistency */
 887                 break;
 888         }
 889         RETURN(rrc);
 890 }
 891 #undef apick_fn
 892 #undef apick_fmt
 893
 894 #define apick_fn "kgnilnd_ep_unbind"
 895 #define apick_fmt "0x%p"
 896 static inline gni_return_t kgnilnd_ep_unbind(
 897                 IN gni_ep_handle_t      ep_hndl
 898                 )
 899 {
 900         gni_return_t rrc;
 901
 902         /* Error injection */
 903         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_UNBIND)) {
 904                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 905         } else {
 906                 rrc = gni_ep_unbind(
 907                         ep_hndl);
 908         }
 909
 910         switch (rrc)  {
 911         /* both of these are OK, upper SW needs to handle */
 912         case GNI_RC_NOT_DONE:
 913         case GNI_RC_SUCCESS:
 914                 break;
 915         case GNI_RC_INVALID_PARAM:
 916                 GNILND_API_SWBUG(
 917                         ep_hndl);
 918                 break;
 919         default:
 920                 GNILND_API_RC_LBUG(
 921                         ep_hndl);
 922
 923                 /* LBUG never returns, but just for style and consistency */
 924                 break;
 925         }
 926         RETURN(rrc);
 927 }
 928 #undef apick_fn
 929 #undef apick_fmt
 930
 931 #define apick_fn "kgnilnd_ep_destroy"
 932 #define apick_fmt "0x%p"
 933 static inline gni_return_t kgnilnd_ep_destroy(
 934                 IN gni_ep_handle_t      ep_hndl
 935                 )
 936 {
 937         gni_return_t rrc;
 938
 939         /* Error injection */
 940         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_DESTROY)) {
 941                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 942         } else {
 943                 rrc = gni_ep_destroy(
 944                         ep_hndl);
 945         }
 946
 947         switch (rrc)  {
 948         case GNI_RC_SUCCESS:
 949                 break;
 950         case GNI_RC_INVALID_PARAM:
 951                 GNILND_API_SWBUG(
 952                         ep_hndl);
 953                 break;
 954         default:
 955                 GNILND_API_RC_LBUG(
 956                         ep_hndl);
 957
 958                 /* LBUG never returns, but just for style and consistency */
 959                 break;
 960         }
 961         RETURN(rrc);
 962 }
 963 #undef apick_fn
 964 #undef apick_fmt
 965
 966 #define apick_fn "kgnilnd_ep_postdata_w_id"
 967 #define apick_fmt "0x%p, 0x%p, %d, 0x%p, %d, %llu"
 968 static inline gni_return_t kgnilnd_ep_postdata_w_id(
 969                 IN gni_ep_handle_t ep_hndl,
 970                 IN void            *in_data,
 971                 IN uint16_t        data_len,
 972                 IN void            *out_buf,
 973                 IN uint16_t        buf_size,
 974                 IN uint64_t        datagram_id
 975                 )
 976 {
 977         gni_return_t rrc;
 978
 979         /* Error injection */
 980         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_POST)) {
 981                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_SIZE_ERROR;
 982         } else {
 983                 rrc = gni_ep_postdata_w_id(
 984                         ep_hndl, in_data, data_len, out_buf, buf_size,
 985                         datagram_id);
 986         }
 987
 988         switch (rrc)  {
 989         case GNI_RC_SUCCESS:
 990         case GNI_RC_ERROR_NOMEM:
 991         case GNI_RC_ERROR_RESOURCE:
 992                 break;
 993         case GNI_RC_INVALID_PARAM:
 994         case GNI_RC_SIZE_ERROR:
 995                 GNILND_API_SWBUG(
 996                         ep_hndl, in_data, data_len, out_buf, buf_size,
 997                         datagram_id);
 998                 break;
 999         default:
1000                 GNILND_API_RC_LBUG(
1001                         ep_hndl, in_data, data_len, out_buf, buf_size,
1002                         datagram_id);
1003
1004                 /* LBUG never returns, but just for style and consistency */
1005                 break;
1006         }
1007         RETURN(rrc);
1008 }
1009 #undef apick_fn
1010 #undef apick_fmt
1011
1012 #define apick_fn "kgnilnd_ep_postdata_test_by_id"
1013 #define apick_fmt "0x%p, %llu, 0x%p, 0x%p, 0x%p"
1014 static inline gni_return_t kgnilnd_ep_postdata_test_by_id(
1015                 IN gni_ep_handle_t      ep_hndl,
1016                 IN uint64_t             datagram_id,
1017                 OUT gni_post_state_t    *post_state,
1018                 OUT uint32_t            *remote_addr,
1019                 OUT uint32_t            *remote_id
1020                 )
1021 {
1022         gni_return_t rrc;
1023
1024         /* Error injection */
1025         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_TEST)) {
1026                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_NOMEM;
1027         } else {
1028                 rrc = gni_ep_postdata_test_by_id(
1029                         ep_hndl, datagram_id, post_state, remote_addr,
1030                         remote_id);
1031
1032                 /* we want to lie, but we need to do the actual work first
1033                  * so we don't keep getting the event saying a dgram is ready */
1034                 if (rrc == GNI_RC_SUCCESS && CFS_FAIL_CHECK(CFS_FAIL_GNI_DG_TERMINATE)) {
1035                         /* don't use fail_val, allows us to do FAIL_SOME */
1036                         *post_state = GNI_POST_TERMINATED;
1037                 }
1038         }
1039
1040         switch (rrc)  {
1041         case GNI_RC_SUCCESS:
1042         case GNI_RC_NO_MATCH:
1043                 break;
1044         case GNI_RC_SIZE_ERROR:
1045         case GNI_RC_INVALID_PARAM:
1046                 GNILND_API_SWBUG(
1047                         ep_hndl, datagram_id, post_state, remote_addr,
1048                         remote_id);
1049                 break;
1050         case GNI_RC_ERROR_NOMEM:
1051                 GNILND_API_RESOURCE(
1052                         ep_hndl, datagram_id, post_state, remote_addr,
1053                         remote_id);
1054                 break;
1055         default:
1056                 GNILND_API_RC_LBUG(
1057                         ep_hndl, datagram_id, post_state, remote_addr,
1058                         remote_id);
1059
1060                 /* LBUG never returns, but just for style and consistency */
1061                 break;
1062         }
1063         RETURN(rrc);
1064 }
1065 #undef apick_fn
1066 #undef apick_fmt
1067
1068 #define apick_fn "kgnilnd_ep_postdata_cancel_by_id"
1069 #define apick_fmt "0x%p, %llu"
1070 static inline gni_return_t kgnilnd_ep_postdata_cancel_by_id(
1071                 IN gni_ep_handle_t      ep_hndl,
1072                 IN uint64_t             datagram_id
1073                 )
1074 {
1075         gni_return_t rrc;
1076
1077         /* no error injection as the only thing we'd do is LBUG */
1078
1079         rrc = gni_ep_postdata_cancel_by_id(
1080                 ep_hndl, datagram_id);
1081
1082         switch (rrc)  {
1083         case GNI_RC_SUCCESS:
1084         case GNI_RC_NO_MATCH:
1085                 break;
1086         case GNI_RC_INVALID_PARAM:
1087                 GNILND_API_SWBUG(
1088                         ep_hndl, datagram_id);
1089                 break;
1090         default:
1091                 GNILND_API_RC_LBUG(
1092                         ep_hndl, datagram_id);
1093
1094                 /* LBUG never returns, but just for style and consistency */
1095                 break;
1096         }
1097         RETURN(rrc);
1098 }
1099 #undef apick_fn
1100 #undef apick_fmt
1101
1102 #define apick_fn "kgnilnd_postdata_probe_by_id"
1103 #define apick_fmt "0x%p, 0x%p"
1104 static inline gni_return_t kgnilnd_postdata_probe_by_id(
1105                 IN gni_nic_handle_t    nic_hndl,
1106                 OUT uint64_t          *datagram_id
1107                 )
1108 {
1109         gni_return_t rrc;
1110
1111         /* Error injection */
1112         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PROBE)) {
1113                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NO_MATCH;
1114         } else {
1115                 rrc = gni_postdata_probe_by_id(
1116                         nic_hndl, datagram_id);
1117         }
1118
1119         switch (rrc)  {
1120         case GNI_RC_SUCCESS:
1121         case GNI_RC_NO_MATCH:
1122                 break;
1123         case GNI_RC_INVALID_PARAM:
1124                 GNILND_API_SWBUG(
1125                         nic_hndl, datagram_id);
1126                 break;
1127         default:
1128                 GNILND_API_RC_LBUG(
1129                         nic_hndl, datagram_id);
1130
1131                 /* LBUG never returns, but just for style and consistency */
1132                 break;
1133         }
1134         RETURN(rrc);
1135 }
1136 #undef apick_fn
1137 #undef apick_fmt
1138
1139 #define apick_fn "kgnilnd_postdata_probe_wait_by_id"
1140 #define apick_fmt "0x%p, %d, 0x%p"
1141 static inline gni_return_t kgnilnd_postdata_probe_wait_by_id(
1142                 IN gni_nic_handle_t nic_hndl,
1143                 IN uint32_t         timeout,
1144                 OUT uint64_t        *datagram_id
1145                 )
1146 {
1147         gni_return_t rrc;
1148
1149         /* Error injection */
1150         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PROBE_WAIT)) {
1151                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_TIMEOUT;
1152         } else {
1153                 rrc = gni_postdata_probe_wait_by_id(
1154                         nic_hndl, timeout, datagram_id);
1155         }
1156
1157         switch (rrc)  {
1158         case GNI_RC_SUCCESS:
1159         case GNI_RC_TIMEOUT:
1160                 break;
1161         case GNI_RC_INVALID_PARAM:
1162                 GNILND_API_SWBUG(
1163                         nic_hndl, timeout, datagram_id);
1164                 break;
1165         default:
1166                 GNILND_API_RC_LBUG(
1167                         nic_hndl, timeout, datagram_id);
1168
1169                 /* LBUG never returns, but just for style and consistency */
1170                 break;
1171         }
1172         RETURN(rrc);
1173 }
1174 #undef apick_fn
1175 #undef apick_fmt
1176
1177 #define apick_fn "kgnilnd_post_rdma"
1178 #define apick_fmt "0x%p, 0x%p"
1179 static inline gni_return_t kgnilnd_post_rdma(
1180                 IN gni_ep_handle_t               ep_hndl,
1181                 IN gni_post_descriptor_t        *post_descr
1182                 )
1183 {
1184         gni_return_t rrc;
1185
1186         /* Error injection */
1187         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_POST_RDMA)) {
1188                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
1189         } else {
1190                 rrc = gni_post_rdma(
1191                         ep_hndl, post_descr);
1192         }
1193
1194         switch (rrc)  {
1195         case GNI_RC_SUCCESS:
1196                 break;
1197         case GNI_RC_ALIGNMENT_ERROR:
1198         case GNI_RC_INVALID_PARAM:
1199                 GNILND_API_SWBUG(
1200                         ep_hndl, post_descr);
1201                 break;
1202         case GNI_RC_ERROR_RESOURCE:
1203                 CDEBUG(D_NET, "no resources for kgnilnd_post_rdma (0x%p, 0x%p)"
1204                         " rc %s\n", ep_hndl, post_descr,
1205                         kgnilnd_api_rc2str(rrc));
1206                 break;
1207         default:
1208                 GNILND_API_RC_LBUG(
1209                         ep_hndl, post_descr);
1210
1211                 /* LBUG never returns, but just for style and consistency */
1212                 break;
1213         }
1214         RETURN(rrc);
1215 }
1216 #undef apick_fn
1217 #undef apick_fmt
1218
1219 #define apick_fn "kgnilnd_get_completed"
1220 #define apick_fmt "0x%p,%#llx,0x%p"
1221 static inline gni_return_t kgnilnd_get_completed(
1222                 IN gni_cq_handle_t              cq_hndl,
1223                 IN gni_cq_entry_t               event_data,
1224                 OUT gni_post_descriptor_t       **post_descr
1225                 )
1226 {
1227         gni_return_t rrc;
1228
1229
1230         rrc = gni_get_completed(cq_hndl, event_data, post_descr);
1231
1232         switch (rrc)  {
1233         case GNI_RC_TRANSACTION_ERROR:
1234         case GNI_RC_SUCCESS:
1235                 break;
1236         case GNI_RC_DESCRIPTOR_ERROR:
1237         case GNI_RC_INVALID_PARAM:
1238                 GNILND_API_SWBUG(cq_hndl, event_data, post_descr);
1239                 break;
1240         default:
1241                 GNILND_API_RC_LBUG(cq_hndl, event_data, post_descr);
1242                 /* LBUG never returns, but just for style and consistency */
1243                 break;
1244         }
1245
1246         /* Error injection - we need a valid desc, so let kgni give us one
1247          * - then we lie  */
1248         if (rrc == GNI_RC_SUCCESS &&
1249             (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED))) {
1250                 /* We only trigger TRANSACTION_ERROR for now */
1251                 gni_post_descriptor_t *desc;
1252                 rrc = GNI_RC_TRANSACTION_ERROR;
1253                 desc = *post_descr;
1254                 desc->status = rrc;
1255                 /* recoverable decision made from cfs_fail_val in
1256                  *  kgnilnd_cq_error_str and
1257                  *  kgnilnd_cq_error_recoverable */
1258         }
1259         RETURN(rrc);
1260 }
1261 #undef apick_fn
1262 #undef apick_fmt
1263
1264 #define apick_fn "kgnilnd_cq_error_str"
1265 #define apick_fmt "%#llx,0x%p,%d"
1266 static inline gni_return_t kgnilnd_cq_error_str(
1267                 IN gni_cq_entry_t       entry,
1268                 IN void                *buffer,
1269                 IN uint32_t             len
1270                 )
1271 {
1272         gni_return_t rrc;
1273
1274         /* Error injection - set string if we injected a
1275          *  TRANSACTION_ERROR earlier */
1276         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED)) {
1277                 /* if we just set persistent error, we can't ever
1278                  * break in via ssh to clear, so use a count > 10 to indicate fatal */
1279                 sprintf(buffer, "INJECT:%s", cfs_fail_val > 10 ?
1280                         "FATAL" : "RECOVERABLE");
1281                 rrc = GNI_RC_SUCCESS;
1282         } else {
1283                 rrc = gni_cq_error_str(
1284                         entry, buffer, len);
1285         }
1286
1287         switch (rrc)  {
1288         case GNI_RC_SUCCESS:
1289                 break;
1290         case GNI_RC_SIZE_ERROR:
1291         case GNI_RC_INVALID_PARAM:
1292                 GNILND_API_SWBUG(
1293                         entry, buffer, len);
1294                 /* give them something to use */
1295                 snprintf(buffer, len, "UNDEF:UNDEF");
1296                 break;
1297         default:
1298                 GNILND_API_RC_LBUG(
1299                         entry, buffer, len);
1300
1301                 /* LBUG never returns, but just for style and consistency */
1302                 break;
1303         }
1304         RETURN(rrc);
1305 }
1306 #undef apick_fn
1307 #undef apick_fmt
1308
1309 #define apick_fn "kgnilnd_cq_error_recoverable"
1310 #define apick_fmt "%#llx,0x%p"
1311 static inline gni_return_t kgnilnd_cq_error_recoverable(
1312                 IN gni_cq_entry_t       entry,
1313                 IN uint32_t            *recoverable
1314                 )
1315 {
1316         gni_return_t rrc;
1317
1318         /* Error injection - set string if we injected a
1319          *  TRANSACTION_ERROR earlier */
1320         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED)) {
1321                 *recoverable = cfs_fail_val > 10 ? 0 : 1;
1322                 rrc = GNI_RC_SUCCESS;
1323         } else {
1324                 rrc = gni_cq_error_recoverable(
1325                         entry, recoverable);
1326         }
1327
1328         switch (rrc)  {
1329         case GNI_RC_SUCCESS:
1330                 break;
1331         case GNI_RC_INVALID_STATE:
1332         case GNI_RC_INVALID_PARAM:
1333                 GNILND_API_SWBUG(
1334                         entry, recoverable);
1335                 *recoverable = 0;
1336                 break;
1337         default:
1338                 GNILND_API_RC_LBUG(
1339                         entry, recoverable);
1340
1341                 /* LBUG never returns, but just for style and consistency */
1342                 break;
1343         }
1344         RETURN(rrc);
1345 }
1346 #undef apick_fn
1347 #undef apick_fmt
1348
1349 #define apick_fn "kgnilnd_mem_register_segments"
1350 #define apick_fmt "0x%p,0x%p,%u,0x%p,%x,0x%p"
1351 static inline gni_return_t
1352 kgnilnd_mem_register_segments(
1353                 IN gni_nic_handle_t     nic_hndl,
1354                 IN gni_mem_segment_t    *mem_segments,
1355                 IN uint32_t             segments_cnt,
1356                 IN gni_cq_handle_t      dst_cq_hndl,
1357                 IN uint32_t             flags,
1358                 OUT gni_mem_handle_t    *mem_hndl
1359                 )
1360 {
1361         gni_return_t rrc;
1362
1363         /* Error injection */
1364         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PHYS_MAP)) {
1365                 rrc = GNI_RC_ERROR_RESOURCE;
1366         } else {
1367                 rrc = gni_mem_register_segments(
1368                         nic_hndl, mem_segments, segments_cnt,
1369                         dst_cq_hndl, flags, mem_hndl);
1370         }
1371
1372         switch (rrc)  {
1373         case GNI_RC_SUCCESS:
1374         case GNI_RC_ERROR_RESOURCE:
1375                 break;
1376         case GNI_RC_INVALID_PARAM:
1377                 GNILND_API_SWBUG(
1378                         nic_hndl, mem_segments, segments_cnt,
1379                         dst_cq_hndl, flags, mem_hndl);
1380                 break;
1381         default:
1382                 GNILND_API_RC_LBUG(
1383                         nic_hndl, mem_segments, segments_cnt,
1384                         dst_cq_hndl, flags, mem_hndl);
1385
1386                 /* LBUG never returns, but just for style and consistency */
1387                 break;
1388         }
1389         RETURN(rrc);
1390 }
1391 #undef apick_fn
1392 #undef apick_fmt
1393
1394 #define apick_fn "kgnilnd_mem_register"
1395 #define apick_fmt "0x%p,%#llx,%#llx0x%p,%u,0x%p"
1396 static inline gni_return_t kgnilnd_mem_register(
1397                 IN gni_nic_handle_t     nic_hndl,
1398                 IN uint64_t             address,
1399                 IN uint64_t             length,
1400                 IN gni_cq_handle_t      dst_cq_hndl,
1401                 IN uint32_t             flags,
1402                 OUT gni_mem_handle_t    *mem_hndl
1403                 )
1404 {
1405         gni_return_t rrc;
1406
1407         /* Error injection */
1408         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_MAP)) {
1409                 rrc = GNI_RC_ERROR_RESOURCE;
1410         } else if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_SMALL_MAP) &&
1411                    length <= *kgnilnd_tunables.kgn_max_immediate) {
1412                 rrc = GNI_RC_INVALID_PARAM;
1413         } else {
1414                 rrc = gni_mem_register(
1415                         nic_hndl, address, length,
1416                         dst_cq_hndl, flags, mem_hndl);
1417         }
1418
1419         /* gni_mem_register may return GNI_RC_ERROR_NOMEM under memory
1420          * pressure but the upper layers only know about resource errors
1421          */
1422         if (rrc == GNI_RC_ERROR_NOMEM) {
1423                 rrc = GNI_RC_ERROR_RESOURCE;
1424         }
1425
1426         switch (rrc)  {
1427         case GNI_RC_SUCCESS:
1428         case GNI_RC_ERROR_RESOURCE:
1429                 break;
1430         case GNI_RC_INVALID_PARAM:
1431                 GNILND_API_SWBUG(
1432                         nic_hndl, address, length,
1433                         dst_cq_hndl, flags, mem_hndl);
1434                 break;
1435         default:
1436                 GNILND_API_RC_LBUG(
1437                         nic_hndl, address, length,
1438                         dst_cq_hndl, flags, mem_hndl);
1439
1440                 /* LBUG never returns, but just for style and consistency */
1441                 break;
1442         }
1443         RETURN(rrc);
1444 }
1445 #undef apick_fn
1446 #undef apick_fmt
1447
1448 #define apick_fn "kgnilnd_mem_deregister"
1449 #define apick_fmt "0x%p,0x%p,%d"
1450 static inline gni_return_t kgnilnd_mem_deregister(
1451                 IN gni_nic_handle_t     nic_hndl,
1452                 IN gni_mem_handle_t     *mem_hndl,
1453                 IN int                  hold_timeout
1454                 )
1455 {
1456         gni_return_t rrc;
1457
1458         /* Error injection */
1459         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_UNMAP)) {
1460                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
1461         } else {
1462                 rrc = gni_mem_deregister(
1463                         nic_hndl, mem_hndl, hold_timeout);
1464         }
1465
1466         switch (rrc)  {
1467         case GNI_RC_SUCCESS:
1468                 break;
1469         case GNI_RC_INVALID_PARAM:
1470                 GNILND_API_SWBUG(
1471                         nic_hndl, mem_hndl, hold_timeout);
1472                 break;
1473         default:
1474                 GNILND_API_RC_LBUG(
1475                         nic_hndl, mem_hndl, hold_timeout);
1476
1477                 /* LBUG never returns, but just for style and consistency */
1478                 break;
1479         }
1480         RETURN(rrc);
1481 }
1482 #undef apick_fn
1483 #undef apick_fmt
1484
1485 #define apick_fn "kgnilnd_mem_mdd_release"
1486 #define apick_fmt "0x%p,0x%p"
1487 static inline gni_return_t kgnilnd_mem_mdd_release(
1488                 IN gni_nic_handle_t     nic_hndl,
1489                 IN gni_mem_handle_t     *mem_hndl
1490                 )
1491 {
1492         gni_return_t rrc;
1493
1494         /* Error injection */
1495         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_MDD_RELEASE)) {
1496                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NO_MATCH;
1497         } else {
1498                 rrc = gni_mem_mdd_release(
1499                         nic_hndl, mem_hndl);
1500         }
1501
1502         switch (rrc)  {
1503         case GNI_RC_SUCCESS:
1504         case GNI_RC_NO_MATCH:
1505                 break;
1506         default:
1507                 GNILND_API_RC_LBUG(
1508                         nic_hndl, mem_hndl);
1509
1510                 /* LBUG never returns, but just for style and consistency */
1511                 break;
1512         }
1513         RETURN(rrc);
1514 }
1515 #undef apick_fn
1516 #undef apick_fmt
1517
1518 #endif /* _GNILND_API_WRAP_H */