lnet/klnds/gnilnd/gnilnd_api_wrap.h

   1 /*
   2  * Copyright (C) 2009-2012 Cray, Inc.
   3  *
   4  *   Author: Nic Henke <nic@cray.com>
   5  *
   6  *   This file is part of Lustre, http://www.lustre.org.
   7  *
   8  *   Lustre is free software; you can redistribute it and/or
   9  *   modify it under the terms of version 2 of the GNU General Public
  10  *   License as published by the Free Software Foundation.
  11  *
  12  *   Lustre is distributed in the hope that it will be useful,
  13  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *   GNU General Public License for more details.
  16  *
  17  *   You should have received a copy of the GNU General Public License
  18  *   along with Lustre; if not, write to the Free Software
  19  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  20  *
  21  */
  22 #ifndef _GNILND_API_WRAP_H
  23 #define _GNILND_API_WRAP_H
  24
  25 /* LNet is allocated failure locations 0xe000 to 0xffff */
  26
  27 /* GNILND has 0xf0XX */
  28 #define CFS_FAIL_GNI                    0xf000
  29 #define CFS_FAIL_GNI_PHYS_MAP           0xf001
  30 #define CFS_FAIL_GNI_VIRT_MAP           0xf002
  31 #define CFS_FAIL_GNI_GET_UNMAP          0xf003
  32 #define CFS_FAIL_GNI_PUT_UNMAP          0xf004
  33 #define CFS_FAIL_GNI_MAP_TX             0xf005
  34 #define CFS_FAIL_GNI_SMSG_SEND          0xf006
  35 #define CFS_FAIL_GNI_CLOSE_SEND         0xf007
  36 #define CFS_FAIL_GNI_CDM_CREATE         0xf008
  37 #define CFS_FAIL_GNI_CDM_DESTROY        0xf009
  38 #define CFS_FAIL_GNI_CDM_ATTACH         0xf00a
  39 #define CFS_FAIL_GNI_CQ_CREATE          0xf00b
  40 #define CFS_FAIL_GNI_CQ_DESTROY         0xf00c
  41 #define CFS_FAIL_GNI_EP_BIND            0xf00d
  42 #define CFS_FAIL_GNI_EP_UNBIND          0xf00e
  43 #define CFS_FAIL_GNI_EP_SET_EVDATA      0xf00f
  44 #define CFS_FAIL_GNI_SMSG_INIT          0xf010
  45 #define CFS_FAIL_GNI_SMSG_RELEASE       0xf011
  46 #define CFS_FAIL_GNI_POST_RDMA          0xf012
  47 #define CFS_FAIL_GNI_GET_COMPLETED      0xf013
  48 #define CFS_FAIL_GNI_EP_DESTROY         0xf015
  49 #define CFS_FAIL_GNI_VIRT_UNMAP         0xf016
  50 #define CFS_FAIL_GNI_MDD_RELEASE        0xf017
  51 #define CFS_FAIL_GNI_NOOP_SEND          0xf018
  52 #define CFS_FAIL_GNI_ERR_SUBSCRIBE      0xf01a
  53 #define CFS_FAIL_GNI_QUIESCE_RACE       0xf01b
  54 #define CFS_FAIL_GNI_DG_TERMINATE       0xf01c
  55 #define CFS_FAIL_GNI_REG_QUIESCE        0xf01d
  56 #define CFS_FAIL_GNI_IN_QUIESCE         0xf01e
  57 #define CFS_FAIL_GNI_DELAY_RDMA         0xf01f
  58 #define CFS_FAIL_GNI_SR_DOWN_RACE       0xf020
  59 #define CFS_FAIL_GNI_ALLOC_TX           0xf021
  60 #define CFS_FAIL_GNI_FMABLK_AVAIL       0xf022
  61 #define CFS_FAIL_GNI_EP_CREATE          0xf023
  62 #define CFS_FAIL_GNI_CQ_GET_EVENT       0xf024
  63 #define CFS_FAIL_GNI_PROBE              0xf025
  64 #define CFS_FAIL_GNI_EP_TEST            0xf026
  65 #define CFS_FAIL_GNI_CONNREQ_DROP       0xf027
  66 #define CFS_FAIL_GNI_CONNREQ_PROTO      0xf028
  67 #define CFS_FAIL_GNI_CONND_PILEUP       0xf029
  68 #define CFS_FAIL_GNI_PHYS_SETUP         0xf02a
  69 #define CFS_FAIL_GNI_FIND_TARGET        0xf02b
  70 #define CFS_FAIL_GNI_WC_DGRAM_FREE      0xf02c
  71 #define CFS_FAIL_GNI_DROP_CLOSING       0xf02d
  72 #define CFS_FAIL_GNI_RX_CLOSE_CLOSING   0xf02e
  73 #define CFS_FAIL_GNI_RX_CLOSE_CLOSED    0xf02f
  74 #define CFS_FAIL_GNI_EP_POST            0xf030
  75 #define CFS_FAIL_GNI_PACK_SRCNID        0xf031
  76 #define CFS_FAIL_GNI_PACK_DSTNID        0xf032
  77 #define CFS_FAIL_GNI_PROBE_WAIT         0xf033
  78 #define CFS_FAIL_GNI_SMSG_CKSUM1        0xf034
  79 #define CFS_FAIL_GNI_SMSG_CKSUM2        0xf035
  80 #define CFS_FAIL_GNI_SMSG_CKSUM3        0xf036
  81 #define CFS_FAIL_GNI_DROP_DESTROY_EP    0xf037
  82 #define CFS_FAIL_GNI_SMSG_GETNEXT       0xf038
  83 #define CFS_FAIL_GNI_FINISH_PURG        0xf039
  84 #define CFS_FAIL_GNI_PURG_REL_DELAY     0xf03a
  85 #define CFS_FAIL_GNI_DONT_NOTIFY        0xf03b
  86 #define CFS_FAIL_GNI_VIRT_SMALL_MAP     0xf03c
  87 #define CFS_FAIL_GNI_DELAY_RDMAQ        0xf03d
  88 #define CFS_FAIL_GNI_PAUSE_SHUTDOWN     0xf03e
  89 #define CFS_FAIL_GNI_PAUSE_DGRAM_COMP   0xf03f
  90 #define CFS_FAIL_GNI_NET_LOOKUP         0xf040
  91 #define CFS_FAIL_GNI_RECV_TIMEOUT       0xf041
  92 #define CFS_FAIL_GNI_SEND_TIMEOUT       0xf042
  93 #define CFS_FAIL_GNI_ONLY_NOOP          0xf043
  94 #define CFS_FAIL_GNI_FINISH_PURG2       0xf044
  95 #define CFS_FAIL_GNI_RACE_RESET         0xf045
  96 #define CFS_FAIL_GNI_GNP_CONNECTING1    0xf046
  97 #define CFS_FAIL_GNI_GNP_CONNECTING2    0xf047
  98 #define CFS_FAIL_GNI_GNP_CONNECTING3    0xf048
  99 #define CFS_FAIL_GNI_SCHEDULE_COMPLETE  0xf049
 100 #define CFS_FAIL_GNI_PUT_ACK_AGAIN      0xf050
 101 #define CFS_FAIL_GNI_GET_REQ_AGAIN      0xf051
 102 #define CFS_FAIL_GNI_SCHED_DEADLINE     0xf052
 103 #define CFS_FAIL_GNI_DGRAM_DEADLINE     0xf053
 104 #define CFS_FAIL_GNI_DGRAM_DROP_TX      0xf054
 105
 106 extern uint32_t kgni_driver_version;
 107
 108 static inline void
 109 kgnilnd_check_kgni_version(void)
 110 {
 111         uint32_t *kdv;
 112
 113         kgnilnd_data.kgn_enable_gl_mutex = 1;
 114         kdv = symbol_get(kgni_driver_version);
 115         if (!kdv) {
 116                 LCONSOLE_INFO("Not using thread safe locking -"
 117                         " no symbol kgni_driver_version\n");
 118                 return;
 119         }
 120
 121         /* Thread-safe kgni implemented in minor ver 0x44/45, code rev 0xb9 */
 122         if (*kdv < GNI_VERSION_CHECK(0, GNILND_KGNI_TS_MINOR_VER, 0xb9)) {
 123                 symbol_put(kgni_driver_version);
 124                 LCONSOLE_INFO("Not using thread safe locking, gni version 0x%x,"
 125                         " need >= 0x%x\n", *kdv,
 126                         GNI_VERSION_CHECK(0, GNILND_KGNI_TS_MINOR_VER, 0xb9));
 127                 return;
 128         }
 129
 130         symbol_put(kgni_driver_version);
 131         /* Use thread-safe locking */
 132         kgnilnd_data.kgn_enable_gl_mutex = 0;
 133 }
 134
 135 /* helper macros */
 136 extern void
 137 _kgnilnd_api_rc_lbug(const char *rcstr, int rc, struct libcfs_debug_msg_data *data,
 138                         const char *fmt, ...)
 139         __attribute__ ((format (printf, 4, 5)));
 140
 141 #define kgnilnd_api_rc_lbug(msgdata, rc, fmt, a...)                             \
 142 do {                                                                            \
 143         CFS_CHECK_STACK(msgdata, D_ERROR, NULL);                                \
 144         /* we don't mask this - it is always at D_ERROR */                      \
 145         _kgnilnd_api_rc_lbug(kgnilnd_api_rc2str(rc), (rc), msgdata, fmt, ##a);  \
 146 } while (0)
 147
 148 #define DO_RETCODE(x) case x: return #x;
 149 static inline const char *
 150 kgnilnd_api_rc2str(gni_return_t rrc)
 151 {
 152
 153         switch (rrc) {
 154                 DO_RETCODE(GNI_RC_SUCCESS)
 155                 DO_RETCODE(GNI_RC_NOT_DONE);
 156                 DO_RETCODE(GNI_RC_INVALID_PARAM);
 157                 DO_RETCODE(GNI_RC_ERROR_RESOURCE);
 158                 DO_RETCODE(GNI_RC_TIMEOUT);
 159                 DO_RETCODE(GNI_RC_PERMISSION_ERROR);
 160                 DO_RETCODE(GNI_RC_DESCRIPTOR_ERROR);
 161                 DO_RETCODE(GNI_RC_ALIGNMENT_ERROR);
 162                 DO_RETCODE(GNI_RC_INVALID_STATE);
 163                 DO_RETCODE(GNI_RC_NO_MATCH);
 164                 DO_RETCODE(GNI_RC_SIZE_ERROR);
 165                 DO_RETCODE(GNI_RC_TRANSACTION_ERROR);
 166                 DO_RETCODE(GNI_RC_ILLEGAL_OP);
 167                 DO_RETCODE(GNI_RC_ERROR_NOMEM);
 168         }
 169         LBUG();
 170 }
 171 #undef DO_RETCODE
 172
 173 /* log an error and LBUG for unhandled rc from gni api function
 174  * the fmt should be something like:
 175  *  gni_api_call(arg1, arg2, arg3)
 176  */
 177
 178 /* apick_fn and apick_fmt should be defined for each site */
 179 #undef apick_fn
 180 #undef apick_fmt
 181
 182 #define GNILND_API_RC_LBUG(args...)                                             \
 183 do {                                                                            \
 184         LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);                     \
 185         kgnilnd_api_rc_lbug(&msgdata, rrc, apick_fn"("apick_fmt")", ##args);    \
 186 } while (0)
 187
 188 #define GNILND_API_SWBUG(args...)                                               \
 189 do {                                                                            \
 190         CERROR("likely SOFTWARE BUG "apick_fn"("apick_fmt") rc %s\n",           \
 191                  ##args, kgnilnd_api_rc2str(rrc));                              \
 192 } while (0)
 193
 194 #define GNILND_API_EINVAL(args...)                                              \
 195 do {                                                                            \
 196         CERROR("invalid parameter to "apick_fn"("apick_fmt") rc %s\n",          \
 197                  ##args, kgnilnd_api_rc2str(rrc));                              \
 198 } while (0)
 199
 200 #define GNILND_API_RESOURCE(args...)                                            \
 201 do {                                                                            \
 202         CERROR("no resources for "apick_fn"("apick_fmt") rc %s\n",              \
 203                 ##args, kgnilnd_api_rc2str(rrc));                               \
 204 } while (0)
 205
 206 #define GNILND_API_BUSY(args...)                                                \
 207 do {                                                                            \
 208         CERROR("resources busy for "apick_fn"("apick_fmt") rc %s\n",            \
 209                 ##args, kgnilnd_api_rc2str(rrc));                               \
 210 } while (0)
 211
 212 #undef DEBUG_SMSG_CREDITS
 213 #ifdef DEBUG_SMSG_CREDITS
 214 #define CRAY_CONFIG_GHAL_GEMINI
 215 #include <gni_priv.h>
 216 #define GNIDBG_SMSG_CREDS(level, conn)                                        \
 217 do {                                                                          \
 218         gni_ep_smsg_mbox_t *smsg = conn->gnc_ephandle->smsg;                  \
 219         CDEBUG(level, "SMSGDBG: conn %p mcred %d/%d bcred %d/%d "             \
 220                 "s_seq %d/%d/%d r_seq %d/%d/%d retr %d\n",                    \
 221                 conn, smsg->mbox_credits, smsg->back_mbox_credits,            \
 222                 smsg->buffer_credits, smsg->back_buffer_credits,              \
 223                 smsg->s_seqno, smsg->s_seqno_back_mbox_credits,               \
 224                 smsg->s_seqno_back_buffer_credits, smsg->r_seqno,             \
 225                 smsg->r_seqno_back_mbox_credits,                              \
 226                 smsg->r_seqno_back_buffer_credits, smsg->retransmit_count);   \
 227 } while (0)
 228 #else
 229 #define GNIDBG_SMSG_CREDS(level, conn) do {} while(0)
 230 #endif
 231
 232 /* these are all wrappers around gni_XXX functions.
 233  * This allows us to handle all the return codes and api checks without
 234  * dirtying up the logic code */
 235
 236 /* TODO: RETURN wrapper that translates integer to GNI API RC string */
 237
 238 #define apick_fn "kgnilnd_cdm_create"
 239 #define apick_fmt "%u, %u, %u, %u, 0x%p"
 240 static inline gni_return_t kgnilnd_cdm_create(
 241                 IN uint32_t             inst_id,
 242                 IN uint8_t              ptag,
 243                 IN uint32_t             cookie,
 244                 IN uint32_t             modes,
 245                 OUT gni_cdm_handle_t    *cdm_hndl
 246                 )
 247 {
 248         gni_return_t rrc;
 249
 250         /* Error injection */
 251         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CDM_CREATE)) {
 252                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 253         } else {
 254                 rrc = gni_cdm_create(inst_id, ptag, cookie, modes, cdm_hndl);
 255         }
 256
 257         switch (rrc)  {
 258         case GNI_RC_SUCCESS:
 259                 break;
 260         case GNI_RC_ERROR_RESOURCE:
 261         case GNI_RC_INVALID_PARAM:
 262                 /* Try to bail gracefully */
 263                 GNILND_API_SWBUG(
 264                         inst_id, ptag, cookie, modes, cdm_hndl);
 265                 break;
 266         default:
 267                 GNILND_API_RC_LBUG(
 268                         inst_id, ptag, cookie, modes, cdm_hndl);
 269
 270                 /* LBUG never returns, but just for style and consistency */
 271                 break;
 272         }
 273         RETURN(rrc);
 274 }
 275
 276 #undef apick_fn
 277 #undef apick_fmt
 278
 279 #define apick_fn "kgnilnd_cdm_attach"
 280 #define apick_fmt "0x%p, %u, 0x%p, 0x%p"
 281 static inline gni_return_t kgnilnd_cdm_attach(
 282                 IN gni_cdm_handle_t     cdm_hndl,
 283                 IN uint32_t             device_id,
 284                 OUT uint32_t            *local_addr,
 285                 OUT gni_nic_handle_t    *nic_hndl
 286                 )
 287 {
 288         gni_return_t rrc;
 289
 290         /* Error injection */
 291         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CDM_ATTACH)) {
 292                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 293         } else {
 294                 rrc = gni_cdm_attach(cdm_hndl, device_id, local_addr, nic_hndl);
 295         }
 296
 297         switch (rrc)  {
 298         case GNI_RC_SUCCESS:
 299                 break;
 300         case GNI_RC_NO_MATCH:
 301         case GNI_RC_INVALID_PARAM:
 302                 GNILND_API_SWBUG(
 303                         cdm_hndl, device_id, local_addr, nic_hndl);
 304                 break;
 305         case GNI_RC_ERROR_RESOURCE:
 306         case GNI_RC_INVALID_STATE:
 307                 GNILND_API_RESOURCE(
 308                         cdm_hndl, device_id, local_addr, nic_hndl);
 309                 break;
 310         default:
 311                 GNILND_API_RC_LBUG(
 312                         cdm_hndl, device_id, local_addr, nic_hndl);
 313
 314                 /* LBUG never returns, but just for style and consistency */
 315                 break;
 316         }
 317         RETURN(rrc);
 318 }
 319 #undef apick_fmt
 320 #undef apick_fn
 321
 322 #define apick_fn "kgnilnd_cdm_destroy"
 323 #define apick_fmt "0x%p"
 324 static inline gni_return_t kgnilnd_cdm_destroy(
 325                 IN gni_cdm_handle_t     cdm_hndl
 326                 )
 327 {
 328         gni_return_t rrc;
 329
 330         /* Error injection */
 331         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_DESTROY)) {
 332                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 333         } else {
 334                 rrc = gni_cdm_destroy(
 335                         cdm_hndl);
 336         }
 337
 338         switch (rrc)  {
 339         case GNI_RC_SUCCESS:
 340                 break;
 341         case GNI_RC_INVALID_PARAM:
 342                 GNILND_API_SWBUG(
 343                         cdm_hndl);
 344                 break;
 345         default:
 346                 GNILND_API_RC_LBUG(
 347                         cdm_hndl);
 348
 349                 /* LBUG never returns, but just for style and consistency */
 350                 break;
 351         }
 352         RETURN(rrc);
 353 }
 354 #undef apick_fn
 355 #undef apick_fmt
 356
 357 #define apick_fn "kgnilnd_subscribe_errors"
 358 #define apick_fmt "0x%p,%x,%u,0x%p,0x%p,0x%p"
 359 static inline gni_return_t kgnilnd_subscribe_errors(
 360                 IN gni_nic_handle_t  nic_handle,
 361                 IN gni_error_mask_t  mask,
 362                 IN uint32_t          EEQ_size,
 363                 IN void              (*EQ_new_event)(gni_err_handle_t),
 364                 IN void              (*app_crit_err)(gni_err_handle_t),
 365                 OUT gni_err_handle_t *err_handle
 366                 )
 367 {
 368         gni_return_t rrc;
 369
 370         /* Error injection */
 371         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_ERR_SUBSCRIBE)) {
 372                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 373         } else {
 374                 rrc = gni_subscribe_errors(
 375                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 376                         err_handle);
 377         }
 378
 379         switch (rrc)  {
 380         case GNI_RC_SUCCESS:
 381                 break;
 382         case GNI_RC_INVALID_PARAM:
 383                 GNILND_API_SWBUG(
 384                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 385                         err_handle);
 386                 break;
 387         case GNI_RC_ERROR_RESOURCE:
 388                 GNILND_API_RESOURCE(
 389                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 390                         err_handle);
 391                 break;
 392         default:
 393                 GNILND_API_RC_LBUG(
 394                         nic_handle, mask, EEQ_size, EQ_new_event, app_crit_err,
 395                         err_handle);
 396                 /* LBUG never returns, but just for style and consistency */
 397                 break;
 398         }
 399         RETURN(rrc);
 400 }
 401 #undef apick_fn
 402 #undef apick_fmt
 403
 404 #define apick_fn "kgnilnd_release_errors"
 405 #define apick_fmt "0x%p"
 406 static inline gni_return_t kgnilnd_release_errors(
 407                 IN gni_err_handle_t err_handle
 408                 )
 409 {
 410         gni_return_t rrc;
 411
 412         rrc = gni_release_errors(
 413                         err_handle);
 414
 415         switch (rrc)  {
 416         case GNI_RC_SUCCESS:
 417                 break;
 418         case GNI_RC_INVALID_PARAM:
 419         case GNI_RC_NOT_DONE:
 420                 GNILND_API_SWBUG(
 421                         err_handle);
 422                 break;
 423         default:
 424                 GNILND_API_RC_LBUG(
 425                         err_handle);
 426                 /* LBUG never returns, but just for style and consistency */
 427                 break;
 428         }
 429         RETURN(rrc);
 430 }
 431 #undef apick_fn
 432 #undef apick_fmt
 433
 434 #define apick_fn "kgnilnd_set_quiesce_callback"
 435 #define apick_fmt "0x%p,0x%p"
 436 static inline gni_return_t kgnilnd_set_quiesce_callback(
 437                 IN gni_nic_handle_t  nic_handle,
 438                 IN void              (*qsce_func)(gni_nic_handle_t, uint64_t msecs)
 439                 )
 440 {
 441         gni_return_t rrc;
 442
 443         /* Error injection */
 444         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_REG_QUIESCE)) {
 445                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 446         } else {
 447                 rrc = gni_set_quiesce_callback(
 448                         nic_handle, qsce_func);
 449         }
 450
 451         switch (rrc)  {
 452         case GNI_RC_SUCCESS:
 453                 break;
 454         case GNI_RC_INVALID_STATE:
 455         case GNI_RC_INVALID_PARAM:
 456                 GNILND_API_SWBUG(
 457                         nic_handle, qsce_func);
 458                 break;
 459         default:
 460                 GNILND_API_RC_LBUG(
 461                         nic_handle, qsce_func);
 462                 /* LBUG never returns, but just for style and consistency */
 463                 break;
 464         }
 465         RETURN(rrc);
 466 }
 467 #undef apick_fn
 468 #undef apick_fmt
 469
 470 #define apick_fn "kgnilnd_get_quiesce_status"
 471 #define apick_fmt "0x%p"
 472 static inline gni_return_t kgnilnd_get_quiesce_status(
 473                 IN gni_nic_handle_t  nic_handle
 474                 )
 475 {
 476         uint32_t rrc;
 477
 478         /* this has weird RC -
 479          * 0 - quiesce not in progress
 480          * 1 - quiesce is turned on
 481         */
 482
 483         /* Error injection */
 484         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_IN_QUIESCE)) {
 485                 rrc = 1;
 486         } else {
 487                 rrc = gni_get_quiesce_status(
 488                         nic_handle);
 489         }
 490
 491         switch (rrc)  {
 492         case 1:
 493         case 0:
 494                 break;
 495         default:
 496                 GNILND_API_RC_LBUG(
 497                         nic_handle);
 498                 /* LBUG never returns, but just for style and consistency */
 499                 break;
 500         }
 501         RETURN(rrc);
 502 }
 503 #undef apick_fn
 504 #undef apick_fmt
 505
 506 #define apick_fn "kgnilnd_cq_create"
 507 #define apick_fmt "0x%p, %u, %u, 0x%p, "LPX64", 0x%p"
 508 static inline gni_return_t kgnilnd_cq_create(
 509                 IN gni_nic_handle_t     nic_hndl,
 510                 IN uint32_t             entry_count,
 511                 IN uint32_t             delay_index,
 512                 IN gni_cq_event_hndlr_f *event_handler,
 513                 IN uint64_t             usr_event_data,
 514                 OUT gni_cq_handle_t     *cq_hndl
 515                 )
 516 {
 517         gni_return_t rrc;
 518
 519         /* Error injection */
 520         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_CREATE)) {
 521                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 522         } else {
 523                 rrc = gni_cq_create(
 524                        nic_hndl, entry_count, delay_index, event_handler,
 525                         usr_event_data, cq_hndl);
 526         }
 527
 528         switch (rrc)  {
 529         case GNI_RC_SUCCESS:
 530                 break;
 531         case GNI_RC_INVALID_PARAM:
 532                 GNILND_API_SWBUG(
 533                         nic_hndl, entry_count, delay_index, event_handler,
 534                         usr_event_data, cq_hndl);
 535                 break;
 536         case GNI_RC_ERROR_RESOURCE:
 537                 GNILND_API_RESOURCE(
 538                         nic_hndl, entry_count, delay_index, event_handler,
 539                         usr_event_data, cq_hndl);
 540                 break;
 541         default:
 542                 GNILND_API_RC_LBUG(
 543                         nic_hndl, entry_count, delay_index, event_handler,
 544                         usr_event_data, cq_hndl);
 545
 546                 /* LBUG never returns, but just for style and consistency */
 547                 break;
 548         }
 549         RETURN(rrc);
 550 }
 551 #undef apick_fn
 552 #undef apick_fmt
 553
 554 #define apick_fn "kgnilnd_cq_destroy"
 555 #define apick_fmt "0x%p"
 556 static inline gni_return_t kgnilnd_cq_destroy(
 557                 IN gni_cq_handle_t cq_hndl
 558                 )
 559 {
 560         gni_return_t rrc;
 561
 562         /* Error injection */
 563         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_CQ_DESTROY)) {
 564                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 565         } else {
 566
 567                 rrc = gni_cq_destroy(
 568                         cq_hndl);
 569         }
 570
 571         switch (rrc)  {
 572         case GNI_RC_SUCCESS:
 573                 break;
 574         case GNI_RC_INVALID_PARAM:
 575                 GNILND_API_SWBUG(
 576                         cq_hndl);
 577                 break;
 578         case GNI_RC_ERROR_RESOURCE:
 579                 GNILND_API_BUSY(
 580                         cq_hndl);
 581                 break;
 582         default:
 583                 GNILND_API_RC_LBUG(
 584                         cq_hndl);
 585
 586                 /* LBUG never returns, but just for style and consistency */
 587                 break;
 588         }
 589         RETURN(rrc);
 590 }
 591 #undef apick_fn
 592 #undef apick_fmt
 593
 594 #define apick_fn "kgnilnd_cq_get_event"
 595 #define apick_fmt "0x%p, 0x%p"
 596 static inline gni_return_t kgnilnd_cq_get_event(
 597                 IN gni_cq_handle_t cq_hndl,
 598                 OUT gni_cq_entry_t *event_data
 599                 )
 600 {
 601         gni_return_t rrc;
 602
 603         /* no error injection - CQs are touchy about the data.
 604          * where appropriate, we'll do this on the CQs that should be able to
 605          * handle the various errors */
 606         rrc = gni_cq_get_event(
 607                         cq_hndl, event_data);
 608
 609         switch (rrc)  {
 610         case GNI_RC_SUCCESS:
 611         case GNI_RC_NOT_DONE:
 612         case GNI_RC_TRANSACTION_ERROR:
 613                 break;
 614         case GNI_RC_ERROR_RESOURCE:
 615                 LASSERTF(GNI_CQ_OVERRUN(*event_data),
 616                          "kgni returned ERROR_RESOURCE but cq_hndl 0x%p is not "
 617                          "overrun\n", cq_hndl);
 618                 break;
 619         case GNI_RC_INVALID_PARAM:
 620                 GNILND_API_SWBUG(
 621                         cq_hndl, event_data);
 622                 break;
 623         default:
 624                 GNILND_API_RC_LBUG(
 625                         cq_hndl, event_data);
 626
 627                 /* LBUG never returns, but just for style and consistency */
 628                 break;
 629         }
 630         return rrc;
 631 }
 632 #undef apick_fn
 633 #undef apick_fmt
 634
 635 #define apick_fn "kgnilnd_smsg_init"
 636 #define apick_fmt "0x%p, 0x%p, 0x%p"
 637 static inline gni_return_t kgnilnd_smsg_init(
 638                 IN gni_ep_handle_t      ep_hndl,
 639                 IN gni_smsg_attr_t      *local_smsg_attr,
 640                 IN gni_smsg_attr_t      *remote_smsg_attr
 641                 )
 642 {
 643         gni_return_t rrc;
 644
 645         /* Error injection */
 646         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_INIT)) {
 647                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_RESOURCE;
 648         } else {
 649                 rrc = gni_smsg_init(
 650                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 651         }
 652
 653         switch (rrc)  {
 654         /* both of these are OK, upper SW needs to handle */
 655         case GNI_RC_SUCCESS:
 656         case GNI_RC_NOT_DONE:
 657                 break;
 658         case GNI_RC_INVALID_PARAM:
 659         case GNI_RC_INVALID_STATE:
 660                 GNILND_API_SWBUG(
 661                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 662                 break;
 663         case GNI_RC_ERROR_RESOURCE:
 664                 GNILND_API_RESOURCE(
 665                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 666                 break;
 667         default:
 668                 GNILND_API_RC_LBUG(
 669                         ep_hndl, local_smsg_attr, remote_smsg_attr);
 670
 671                 /* LBUG never returns, but just for style and consistency */
 672                 break;
 673         }
 674         RETURN(rrc);
 675 }
 676 #undef apick_fn
 677 #undef apick_fmt
 678
 679 #define apick_fn "kgnilnd_smsg_send"
 680 #define apick_fmt "0x%p, 0x%p, %d, 0x%p, %u %u"
 681 static inline gni_return_t kgnilnd_smsg_send(
 682                 IN gni_ep_handle_t      ep_hndl,
 683                 IN void                 *header,
 684                 IN uint32_t             header_length,
 685                 IN void                 *data,
 686                 IN uint32_t             data_length,
 687                 IN uint32_t             msg_id
 688                 )
 689 {
 690         gni_return_t rrc;
 691
 692         /* Error injection */
 693         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_SEND)) {
 694                 if (cfs_fail_loc & CFS_FAIL_RAND) {
 695                         rrc = GNI_RC_NOT_DONE;
 696                 } else {
 697                         rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 698                 }
 699         } else {
 700                 rrc = gni_smsg_send(
 701                         ep_hndl, header, header_length, data, data_length, msg_id);
 702         }
 703
 704         switch (rrc)  {
 705         /* both of these are OK, upper SW needs to handle */
 706         case GNI_RC_SUCCESS:
 707         case GNI_RC_NOT_DONE:
 708                 break;
 709         case GNI_RC_INVALID_PARAM:
 710                 GNILND_API_SWBUG(
 711                         ep_hndl, header, header_length, data, data_length, msg_id);
 712                 break;
 713         case GNI_RC_ERROR_RESOURCE:
 714                 GNILND_API_RESOURCE(
 715                         ep_hndl, header, header_length, data, data_length, msg_id);
 716                 break;
 717         default:
 718                 GNILND_API_RC_LBUG(
 719                         ep_hndl, header, header_length, data, data_length, msg_id);
 720
 721                 /* LBUG never returns, but just for style and consistency */
 722                 break;
 723         }
 724         RETURN(rrc);
 725 }
 726 #undef apick_fn
 727 #undef apick_fmt
 728
 729 #define apick_fn "kgnilnd_smsg_getnext"
 730 #define apick_fmt "0x%p,0x%p"
 731 static inline gni_return_t kgnilnd_smsg_getnext(
 732                 IN gni_ep_handle_t      ep_hndl,
 733                 OUT void                **header
 734                 )
 735 {
 736         gni_return_t rrc;
 737
 738         /* Error injection */
 739         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_RELEASE)) {
 740                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 741         } else {
 742                 rrc = gni_smsg_getnext(
 743                         ep_hndl, header);
 744         }
 745
 746         switch (rrc)  {
 747         /* both of these are OK, upper SW needs to handle */
 748         case GNI_RC_SUCCESS:
 749         case GNI_RC_NOT_DONE:
 750         case GNI_RC_INVALID_STATE:
 751                 break;
 752         case GNI_RC_INVALID_PARAM:
 753                 GNILND_API_SWBUG(
 754                         ep_hndl, header);
 755                 break;
 756         default:
 757                 GNILND_API_RC_LBUG(
 758                         ep_hndl, header);
 759
 760                 /* LBUG never returns, but just for style and consistency */
 761                 break;
 762         }
 763         RETURN(rrc);
 764 }
 765 #undef apick_fn
 766 #undef apick_fmt
 767
 768 #define apick_fn "kgnilnd_smsg_release"
 769 #define apick_fmt "0x%p"
 770 static inline gni_return_t kgnilnd_smsg_release(
 771                 IN gni_ep_handle_t      ep_hndl
 772                 )
 773 {
 774         gni_return_t rrc;
 775
 776         /* Error injection */
 777         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_SMSG_RELEASE)) {
 778                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 779         } else {
 780                 rrc = gni_smsg_release(
 781                         ep_hndl);
 782         }
 783
 784         switch (rrc)  {
 785         /* both of these are OK, upper SW needs to handle */
 786         case GNI_RC_SUCCESS:
 787         case GNI_RC_NOT_DONE:
 788                 break;
 789         case GNI_RC_INVALID_PARAM:
 790                 GNILND_API_SWBUG(
 791                         ep_hndl);
 792                 break;
 793         default:
 794                 GNILND_API_RC_LBUG(
 795                         ep_hndl);
 796
 797                 /* LBUG never returns, but just for style and consistency */
 798                 break;
 799         }
 800         RETURN(rrc);
 801 }
 802 #undef apick_fn
 803 #undef apick_fmt
 804
 805 #define apick_fn "kgnilnd_ep_create"
 806 #define apick_fmt "0x%p, 0x%p, 0x%p"
 807 static inline gni_return_t kgnilnd_ep_create(
 808                 IN gni_nic_handle_t     nic_hndl,
 809                 IN gni_cq_handle_t      src_cq_hndl,
 810                 OUT gni_ep_handle_t     *ep_hndl
 811                 )
 812 {
 813         gni_return_t rrc;
 814
 815         /* error injection */
 816         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_CREATE)) {
 817                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_NOMEM;
 818         } else {
 819                 rrc = gni_ep_create(
 820                         nic_hndl, src_cq_hndl, ep_hndl);
 821         }
 822
 823         switch (rrc)  {
 824         case GNI_RC_SUCCESS:
 825                 break;
 826         case GNI_RC_INVALID_PARAM:
 827                 GNILND_API_SWBUG(
 828                         nic_hndl, src_cq_hndl, ep_hndl);
 829                 break;
 830         case GNI_RC_ERROR_NOMEM:
 831                 GNILND_API_RESOURCE(
 832                         nic_hndl, src_cq_hndl, ep_hndl);
 833                 break;
 834         default:
 835                 GNILND_API_RC_LBUG(
 836                         nic_hndl, src_cq_hndl, ep_hndl);
 837
 838                 /* lbug never returns, but just for style and consistency */
 839                 break;
 840         }
 841         RETURN(rrc);
 842 }
 843 #undef apick_fn
 844 #undef apick_fmt
 845
 846 #define apick_fn "kgnilnd_ep_bind"
 847 #define apick_fmt "0x%p, %x, %x"
 848 static inline gni_return_t kgnilnd_ep_bind(
 849                 IN gni_ep_handle_t      ep_hndl,
 850                 IN uint32_t             remote_addr,
 851                 IN uint32_t             remote_id
 852                 )
 853 {
 854         gni_return_t rrc;
 855
 856         /* error injection */
 857         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_BIND)) {
 858                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 859         } else {
 860                 rrc = gni_ep_bind(
 861                         ep_hndl, remote_addr, remote_id);
 862         }
 863
 864         switch (rrc)  {
 865         /* both of these are ok, upper sw needs to handle */
 866         case GNI_RC_SUCCESS:
 867         case GNI_RC_NOT_DONE:
 868                 break;
 869         case GNI_RC_INVALID_PARAM:
 870                 GNILND_API_SWBUG(
 871                         ep_hndl, remote_addr, remote_id);
 872                 break;
 873         default:
 874                 GNILND_API_RC_LBUG(
 875                         ep_hndl, remote_addr, remote_id);
 876
 877                 /* lbug never returns, but just for style and consistency */
 878                 break;
 879         }
 880         RETURN(rrc);
 881 }
 882 #undef apick_fn
 883 #undef apick_fmt
 884
 885 #define apick_fn "kgnilnd_ep_set_eventdata"
 886 #define apick_fmt "0x%p, %x, %x"
 887 static inline gni_return_t kgnilnd_ep_set_eventdata(
 888                 IN gni_ep_handle_t      ep_hndl,
 889                 IN uint32_t             local_event,
 890                 IN uint32_t             remote_event
 891                 )
 892 {
 893         gni_return_t rrc;
 894
 895         /* Error injection */
 896         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_SET_EVDATA)) {
 897                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
 898         } else {
 899                 rrc = gni_ep_set_eventdata(
 900                         ep_hndl, local_event, remote_event);
 901         }
 902
 903         switch (rrc)  {
 904         case GNI_RC_SUCCESS:
 905                 break;
 906         case GNI_RC_INVALID_PARAM:
 907                 GNILND_API_SWBUG(
 908                         ep_hndl, local_event, remote_event);
 909                 break;
 910         default:
 911                 GNILND_API_RC_LBUG(
 912                         ep_hndl, local_event, remote_event);
 913
 914                 /* LBUG never returns, but just for style and consistency */
 915                 break;
 916         }
 917         RETURN(rrc);
 918 }
 919 #undef apick_fn
 920 #undef apick_fmt
 921
 922 #define apick_fn "kgnilnd_ep_unbind"
 923 #define apick_fmt "0x%p"
 924 static inline gni_return_t kgnilnd_ep_unbind(
 925                 IN gni_ep_handle_t      ep_hndl
 926                 )
 927 {
 928         gni_return_t rrc;
 929
 930         /* Error injection */
 931         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_UNBIND)) {
 932                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 933         } else {
 934                 rrc = gni_ep_unbind(
 935                         ep_hndl);
 936         }
 937
 938         switch (rrc)  {
 939         /* both of these are OK, upper SW needs to handle */
 940         case GNI_RC_NOT_DONE:
 941         case GNI_RC_SUCCESS:
 942                 break;
 943         case GNI_RC_INVALID_PARAM:
 944                 GNILND_API_SWBUG(
 945                         ep_hndl);
 946                 break;
 947         default:
 948                 GNILND_API_RC_LBUG(
 949                         ep_hndl);
 950
 951                 /* LBUG never returns, but just for style and consistency */
 952                 break;
 953         }
 954         RETURN(rrc);
 955 }
 956 #undef apick_fn
 957 #undef apick_fmt
 958
 959 #define apick_fn "kgnilnd_ep_destroy"
 960 #define apick_fmt "0x%p"
 961 static inline gni_return_t kgnilnd_ep_destroy(
 962                 IN gni_ep_handle_t      ep_hndl
 963                 )
 964 {
 965         gni_return_t rrc;
 966
 967         /* Error injection */
 968         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_DESTROY)) {
 969                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NOT_DONE;
 970         } else {
 971                 rrc = gni_ep_destroy(
 972                         ep_hndl);
 973         }
 974
 975         switch (rrc)  {
 976         case GNI_RC_SUCCESS:
 977                 break;
 978         case GNI_RC_INVALID_PARAM:
 979                 GNILND_API_SWBUG(
 980                         ep_hndl);
 981                 break;
 982         default:
 983                 GNILND_API_RC_LBUG(
 984                         ep_hndl);
 985
 986                 /* LBUG never returns, but just for style and consistency */
 987                 break;
 988         }
 989         RETURN(rrc);
 990 }
 991 #undef apick_fn
 992 #undef apick_fmt
 993
 994 #define apick_fn "kgnilnd_ep_postdata_w_id"
 995 #define apick_fmt "0x%p, 0x%p, %d, 0x%p, %d, "LPU64""
 996 static inline gni_return_t kgnilnd_ep_postdata_w_id(
 997                 IN gni_ep_handle_t ep_hndl,
 998                 IN void            *in_data,
 999                 IN uint16_t        data_len,
1000                 IN void            *out_buf,
1001                 IN uint16_t        buf_size,
1002                 IN uint64_t        datagram_id
1003                 )
1004 {
1005         gni_return_t rrc;
1006
1007         /* Error injection */
1008         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_POST)) {
1009                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_SIZE_ERROR;
1010         } else {
1011                 rrc = gni_ep_postdata_w_id(
1012                         ep_hndl, in_data, data_len, out_buf, buf_size,
1013                         datagram_id);
1014         }
1015
1016         switch (rrc)  {
1017         case GNI_RC_SUCCESS:
1018         case GNI_RC_ERROR_NOMEM:
1019         case GNI_RC_ERROR_RESOURCE:
1020                 break;
1021         case GNI_RC_INVALID_PARAM:
1022         case GNI_RC_SIZE_ERROR:
1023                 GNILND_API_SWBUG(
1024                         ep_hndl, in_data, data_len, out_buf, buf_size,
1025                         datagram_id);
1026                 break;
1027         default:
1028                 GNILND_API_RC_LBUG(
1029                         ep_hndl, in_data, data_len, out_buf, buf_size,
1030                         datagram_id);
1031
1032                 /* LBUG never returns, but just for style and consistency */
1033                 break;
1034         }
1035         RETURN(rrc);
1036 }
1037 #undef apick_fn
1038 #undef apick_fmt
1039
1040 #define apick_fn "kgnilnd_ep_postdata_test_by_id"
1041 #define apick_fmt "0x%p, "LPU64", 0x%p, 0x%p, 0x%p"
1042 static inline gni_return_t kgnilnd_ep_postdata_test_by_id(
1043                 IN gni_ep_handle_t      ep_hndl,
1044                 IN uint64_t             datagram_id,
1045                 OUT gni_post_state_t    *post_state,
1046                 OUT uint32_t            *remote_addr,
1047                 OUT uint32_t            *remote_id
1048                 )
1049 {
1050         gni_return_t rrc;
1051
1052         /* Error injection */
1053         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_EP_TEST)) {
1054                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_ERROR_NOMEM;
1055         } else {
1056                 rrc = gni_ep_postdata_test_by_id(
1057                         ep_hndl, datagram_id, post_state, remote_addr,
1058                         remote_id);
1059
1060                 /* we want to lie, but we need to do the actual work first
1061                  * so we don't keep getting the event saying a dgram is ready */
1062                 if (rrc == GNI_RC_SUCCESS && CFS_FAIL_CHECK(CFS_FAIL_GNI_DG_TERMINATE)) {
1063                         /* don't use fail_val, allows us to do FAIL_SOME */
1064                         *post_state = GNI_POST_TERMINATED;
1065                 }
1066         }
1067
1068         switch (rrc)  {
1069         case GNI_RC_SUCCESS:
1070         case GNI_RC_NO_MATCH:
1071                 break;
1072         case GNI_RC_SIZE_ERROR:
1073         case GNI_RC_INVALID_PARAM:
1074                 GNILND_API_SWBUG(
1075                         ep_hndl, datagram_id, post_state, remote_addr,
1076                         remote_id);
1077                 break;
1078         case GNI_RC_ERROR_NOMEM:
1079                 GNILND_API_RESOURCE(
1080                         ep_hndl, datagram_id, post_state, remote_addr,
1081                         remote_id);
1082                 break;
1083         default:
1084                 GNILND_API_RC_LBUG(
1085                         ep_hndl, datagram_id, post_state, remote_addr,
1086                         remote_id);
1087
1088                 /* LBUG never returns, but just for style and consistency */
1089                 break;
1090         }
1091         RETURN(rrc);
1092 }
1093 #undef apick_fn
1094 #undef apick_fmt
1095
1096 #define apick_fn "kgnilnd_ep_postdata_cancel_by_id"
1097 #define apick_fmt "0x%p, "LPU64""
1098 static inline gni_return_t kgnilnd_ep_postdata_cancel_by_id(
1099                 IN gni_ep_handle_t      ep_hndl,
1100                 IN uint64_t             datagram_id
1101                 )
1102 {
1103         gni_return_t rrc;
1104
1105         /* no error injection as the only thing we'd do is LBUG */
1106
1107         rrc = gni_ep_postdata_cancel_by_id(
1108                 ep_hndl, datagram_id);
1109
1110         switch (rrc)  {
1111         case GNI_RC_SUCCESS:
1112         case GNI_RC_NO_MATCH:
1113                 break;
1114         case GNI_RC_INVALID_PARAM:
1115                 GNILND_API_SWBUG(
1116                         ep_hndl, datagram_id);
1117                 break;
1118         default:
1119                 GNILND_API_RC_LBUG(
1120                         ep_hndl, datagram_id);
1121
1122                 /* LBUG never returns, but just for style and consistency */
1123                 break;
1124         }
1125         RETURN(rrc);
1126 }
1127 #undef apick_fn
1128 #undef apick_fmt
1129
1130 #define apick_fn "kgnilnd_postdata_probe_by_id"
1131 #define apick_fmt "0x%p, 0x%p"
1132 static inline gni_return_t kgnilnd_postdata_probe_by_id(
1133                 IN gni_nic_handle_t    nic_hndl,
1134                 OUT uint64_t          *datagram_id
1135                 )
1136 {
1137         gni_return_t rrc;
1138
1139         /* Error injection */
1140         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PROBE)) {
1141                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NO_MATCH;
1142         } else {
1143                 rrc = gni_postdata_probe_by_id(
1144                         nic_hndl, datagram_id);
1145         }
1146
1147         switch (rrc)  {
1148         case GNI_RC_SUCCESS:
1149         case GNI_RC_NO_MATCH:
1150                 break;
1151         case GNI_RC_INVALID_PARAM:
1152                 GNILND_API_SWBUG(
1153                         nic_hndl, datagram_id);
1154                 break;
1155         default:
1156                 GNILND_API_RC_LBUG(
1157                         nic_hndl, datagram_id);
1158
1159                 /* LBUG never returns, but just for style and consistency */
1160                 break;
1161         }
1162         RETURN(rrc);
1163 }
1164 #undef apick_fn
1165 #undef apick_fmt
1166
1167 #define apick_fn "kgnilnd_postdata_probe_wait_by_id"
1168 #define apick_fmt "0x%p, %d, 0x%p"
1169 static inline gni_return_t kgnilnd_postdata_probe_wait_by_id(
1170                 IN gni_nic_handle_t nic_hndl,
1171                 IN uint32_t         timeout,
1172                 OUT uint64_t        *datagram_id
1173                 )
1174 {
1175         gni_return_t rrc;
1176
1177         /* Error injection */
1178         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PROBE_WAIT)) {
1179                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_TIMEOUT;
1180         } else {
1181                 rrc = gni_postdata_probe_wait_by_id(
1182                         nic_hndl, timeout, datagram_id);
1183         }
1184
1185         switch (rrc)  {
1186         case GNI_RC_SUCCESS:
1187         case GNI_RC_TIMEOUT:
1188                 break;
1189         case GNI_RC_INVALID_PARAM:
1190                 GNILND_API_SWBUG(
1191                         nic_hndl, timeout, datagram_id);
1192                 break;
1193         default:
1194                 GNILND_API_RC_LBUG(
1195                         nic_hndl, timeout, datagram_id);
1196
1197                 /* LBUG never returns, but just for style and consistency */
1198                 break;
1199         }
1200         RETURN(rrc);
1201 }
1202 #undef apick_fn
1203 #undef apick_fmt
1204
1205 #define apick_fn "kgnilnd_post_rdma"
1206 #define apick_fmt "0x%p, 0x%p"
1207 static inline gni_return_t kgnilnd_post_rdma(
1208                 IN gni_ep_handle_t               ep_hndl,
1209                 IN gni_post_descriptor_t        *post_descr
1210                 )
1211 {
1212         gni_return_t rrc;
1213
1214         /* Error injection */
1215         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_POST_RDMA)) {
1216                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
1217         } else {
1218                 rrc = gni_post_rdma(
1219                         ep_hndl, post_descr);
1220         }
1221
1222         switch (rrc)  {
1223         case GNI_RC_SUCCESS:
1224                 break;
1225         case GNI_RC_ALIGNMENT_ERROR:
1226         case GNI_RC_INVALID_PARAM:
1227                 GNILND_API_SWBUG(
1228                         ep_hndl, post_descr);
1229                 break;
1230         case GNI_RC_ERROR_RESOURCE:
1231                 CDEBUG(D_NET, "no resources for kgnilnd_post_rdma (0x%p, 0x%p)"
1232                         " rc %s\n", ep_hndl, post_descr,
1233                         kgnilnd_api_rc2str(rrc));
1234                 break;
1235         default:
1236                 GNILND_API_RC_LBUG(
1237                         ep_hndl, post_descr);
1238
1239                 /* LBUG never returns, but just for style and consistency */
1240                 break;
1241         }
1242         RETURN(rrc);
1243 }
1244 #undef apick_fn
1245 #undef apick_fmt
1246
1247 #define apick_fn "kgnilnd_get_completed"
1248 #define apick_fmt "0x%p,"LPX64",0x%p"
1249 static inline gni_return_t kgnilnd_get_completed(
1250                 IN gni_cq_handle_t              cq_hndl,
1251                 IN gni_cq_entry_t               event_data,
1252                 OUT gni_post_descriptor_t       **post_descr
1253                 )
1254 {
1255         gni_return_t rrc;
1256
1257
1258         rrc = gni_get_completed(cq_hndl, event_data, post_descr);
1259
1260         switch (rrc)  {
1261         case GNI_RC_TRANSACTION_ERROR:
1262         case GNI_RC_SUCCESS:
1263                 break;
1264         case GNI_RC_DESCRIPTOR_ERROR:
1265         case GNI_RC_INVALID_PARAM:
1266                 GNILND_API_SWBUG(cq_hndl, event_data, post_descr);
1267                 break;
1268         default:
1269                 GNILND_API_RC_LBUG(cq_hndl, event_data, post_descr);
1270                 /* LBUG never returns, but just for style and consistency */
1271                 break;
1272         }
1273
1274         /* Error injection - we need a valid desc, so let kgni give us one
1275          * - then we lie  */
1276         if (rrc == GNI_RC_SUCCESS &&
1277             (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED))) {
1278                 /* We only trigger TRANSACTION_ERROR for now */
1279                 gni_post_descriptor_t *desc;
1280                 rrc = GNI_RC_TRANSACTION_ERROR;
1281                 desc = *post_descr;
1282                 desc->status = rrc;
1283                 /* recoverable decision made from cfs_fail_val in
1284                  *  kgnilnd_cq_error_str and
1285                  *  kgnilnd_cq_error_recoverable */
1286         }
1287         RETURN(rrc);
1288 }
1289 #undef apick_fn
1290 #undef apick_fmt
1291
1292 #define apick_fn "kgnilnd_cq_error_str"
1293 #define apick_fmt LPX64",0x%p,%d"
1294 static inline gni_return_t kgnilnd_cq_error_str(
1295                 IN gni_cq_entry_t       entry,
1296                 IN void                *buffer,
1297                 IN uint32_t             len
1298                 )
1299 {
1300         gni_return_t rrc;
1301
1302         /* Error injection - set string if we injected a
1303          *  TRANSACTION_ERROR earlier */
1304         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED)) {
1305                 /* if we just set persistent error, we can't ever
1306                  * break in via ssh to clear, so use a count > 10 to indicate fatal */
1307                 sprintf(buffer, "INJECT:%s", cfs_fail_val > 10 ?
1308                         "FATAL" : "RECOVERABLE");
1309                 rrc = GNI_RC_SUCCESS;
1310         } else {
1311                 rrc = gni_cq_error_str(
1312                         entry, buffer, len);
1313         }
1314
1315         switch (rrc)  {
1316         case GNI_RC_SUCCESS:
1317                 break;
1318         case GNI_RC_SIZE_ERROR:
1319         case GNI_RC_INVALID_PARAM:
1320                 GNILND_API_SWBUG(
1321                         entry, buffer, len);
1322                 /* give them something to use */
1323                 snprintf(buffer, len, "UNDEF:UNDEF");
1324                 break;
1325         default:
1326                 GNILND_API_RC_LBUG(
1327                         entry, buffer, len);
1328
1329                 /* LBUG never returns, but just for style and consistency */
1330                 break;
1331         }
1332         RETURN(rrc);
1333 }
1334 #undef apick_fn
1335 #undef apick_fmt
1336
1337 #define apick_fn "kgnilnd_cq_error_recoverable"
1338 #define apick_fmt LPX64",0x%p"
1339 static inline gni_return_t kgnilnd_cq_error_recoverable(
1340                 IN gni_cq_entry_t       entry,
1341                 IN uint32_t            *recoverable
1342                 )
1343 {
1344         gni_return_t rrc;
1345
1346         /* Error injection - set string if we injected a
1347          *  TRANSACTION_ERROR earlier */
1348         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_GET_COMPLETED)) {
1349                 *recoverable = cfs_fail_val > 10 ? 0 : 1;
1350                 rrc = GNI_RC_SUCCESS;
1351         } else {
1352                 rrc = gni_cq_error_recoverable(
1353                         entry, recoverable);
1354         }
1355
1356         switch (rrc)  {
1357         case GNI_RC_SUCCESS:
1358                 break;
1359         case GNI_RC_INVALID_STATE:
1360         case GNI_RC_INVALID_PARAM:
1361                 GNILND_API_SWBUG(
1362                         entry, recoverable);
1363                 *recoverable = 0;
1364                 break;
1365         default:
1366                 GNILND_API_RC_LBUG(
1367                         entry, recoverable);
1368
1369                 /* LBUG never returns, but just for style and consistency */
1370                 break;
1371         }
1372         RETURN(rrc);
1373 }
1374 #undef apick_fn
1375 #undef apick_fmt
1376
1377 #define apick_fn "kgnilnd_mem_register_segments"
1378 #define apick_fmt "0x%p,0x%p,%u,0x%p,%x,0x%p"
1379 static inline gni_return_t
1380 kgnilnd_mem_register_segments(
1381                 IN gni_nic_handle_t     nic_hndl,
1382                 IN gni_mem_segment_t    *mem_segments,
1383                 IN uint32_t             segments_cnt,
1384                 IN gni_cq_handle_t      dst_cq_hndl,
1385                 IN uint32_t             flags,
1386                 OUT gni_mem_handle_t    *mem_hndl
1387                 )
1388 {
1389         gni_return_t rrc;
1390
1391         /* Error injection */
1392         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_PHYS_MAP)) {
1393                 rrc = GNI_RC_ERROR_RESOURCE;
1394         } else {
1395                 rrc = gni_mem_register_segments(
1396                         nic_hndl, mem_segments, segments_cnt,
1397                         dst_cq_hndl, flags, mem_hndl);
1398         }
1399
1400         switch (rrc)  {
1401         case GNI_RC_SUCCESS:
1402         case GNI_RC_ERROR_RESOURCE:
1403                 break;
1404         case GNI_RC_INVALID_PARAM:
1405                 GNILND_API_SWBUG(
1406                         nic_hndl, mem_segments, segments_cnt,
1407                         dst_cq_hndl, flags, mem_hndl);
1408                 break;
1409         default:
1410                 GNILND_API_RC_LBUG(
1411                         nic_hndl, mem_segments, segments_cnt,
1412                         dst_cq_hndl, flags, mem_hndl);
1413
1414                 /* LBUG never returns, but just for style and consistency */
1415                 break;
1416         }
1417         RETURN(rrc);
1418 }
1419 #undef apick_fn
1420 #undef apick_fmt
1421
1422 #define apick_fn "kgnilnd_mem_register"
1423 #define apick_fmt "0x%p,"LPX64","LPX64"0x%p,%u,0x%p"
1424 static inline gni_return_t kgnilnd_mem_register(
1425                 IN gni_nic_handle_t     nic_hndl,
1426                 IN uint64_t             address,
1427                 IN uint64_t             length,
1428                 IN gni_cq_handle_t      dst_cq_hndl,
1429                 IN uint32_t             flags,
1430                 OUT gni_mem_handle_t    *mem_hndl
1431                 )
1432 {
1433         gni_return_t rrc;
1434
1435         /* Error injection */
1436         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_MAP)) {
1437                 rrc = GNI_RC_ERROR_RESOURCE;
1438         } else if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_SMALL_MAP) &&
1439                    length <= *kgnilnd_tunables.kgn_max_immediate) {
1440                 rrc = GNI_RC_INVALID_PARAM;
1441         } else {
1442                 rrc = gni_mem_register(
1443                         nic_hndl, address, length,
1444                         dst_cq_hndl, flags, mem_hndl);
1445         }
1446
1447         switch (rrc)  {
1448         case GNI_RC_SUCCESS:
1449         case GNI_RC_ERROR_RESOURCE:
1450                 break;
1451         case GNI_RC_INVALID_PARAM:
1452                 GNILND_API_SWBUG(
1453                         nic_hndl, address, length,
1454                         dst_cq_hndl, flags, mem_hndl);
1455                 break;
1456         default:
1457                 GNILND_API_RC_LBUG(
1458                         nic_hndl, address, length,
1459                         dst_cq_hndl, flags, mem_hndl);
1460
1461                 /* LBUG never returns, but just for style and consistency */
1462                 break;
1463         }
1464         RETURN(rrc);
1465 }
1466 #undef apick_fn
1467 #undef apick_fmt
1468
1469 #define apick_fn "kgnilnd_mem_deregister"
1470 #define apick_fmt "0x%p,0x%p,%d"
1471 static inline gni_return_t kgnilnd_mem_deregister(
1472                 IN gni_nic_handle_t     nic_hndl,
1473                 IN gni_mem_handle_t     *mem_hndl,
1474                 IN int                  hold_timeout
1475                 )
1476 {
1477         gni_return_t rrc;
1478
1479         /* Error injection */
1480         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_VIRT_UNMAP)) {
1481                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_INVALID_PARAM;
1482         } else {
1483                 rrc = gni_mem_deregister(
1484                         nic_hndl, mem_hndl, hold_timeout);
1485         }
1486
1487         switch (rrc)  {
1488         case GNI_RC_SUCCESS:
1489                 break;
1490         case GNI_RC_INVALID_PARAM:
1491                 GNILND_API_SWBUG(
1492                         nic_hndl, mem_hndl, hold_timeout);
1493                 break;
1494         default:
1495                 GNILND_API_RC_LBUG(
1496                         nic_hndl, mem_hndl, hold_timeout);
1497
1498                 /* LBUG never returns, but just for style and consistency */
1499                 break;
1500         }
1501         RETURN(rrc);
1502 }
1503 #undef apick_fn
1504 #undef apick_fmt
1505
1506 #define apick_fn "kgnilnd_mem_mdd_release"
1507 #define apick_fmt "0x%p,0x%p"
1508 static inline gni_return_t kgnilnd_mem_mdd_release(
1509                 IN gni_nic_handle_t     nic_hndl,
1510                 IN gni_mem_handle_t     *mem_hndl
1511                 )
1512 {
1513         gni_return_t rrc;
1514
1515         /* Error injection */
1516         if (CFS_FAIL_CHECK(CFS_FAIL_GNI_MDD_RELEASE)) {
1517                 rrc = cfs_fail_val ? cfs_fail_val : GNI_RC_NO_MATCH;
1518         } else {
1519                 rrc = gni_mem_mdd_release(
1520                         nic_hndl, mem_hndl);
1521         }
1522
1523         switch (rrc)  {
1524         case GNI_RC_SUCCESS:
1525         case GNI_RC_NO_MATCH:
1526                 break;
1527         default:
1528                 GNILND_API_RC_LBUG(
1529                         nic_hndl, mem_hndl);
1530
1531                 /* LBUG never returns, but just for style and consistency */
1532                 break;
1533         }
1534         RETURN(rrc);
1535 }
1536 #undef apick_fn
1537 #undef apick_fmt
1538
1539 #endif /* _GNILND_API_WRAP_H */