GOTO(failed, rc = -EINVAL);
}
+ rrc = kgnilnd_register_smdd_buf(dev);
+ if (rrc != GNI_RC_SUCCESS) {
+ GOTO(failed, rc = -EINVAL);
+ }
+
RETURN(0);
failed:
atomic_read(&kgnilnd_data.kgn_nthreads) == 0,
"tried to shutdown with threads active\n");
+ if (dev->gnd_smdd_hold_buf) {
+ rrc = kgnilnd_deregister_smdd_buf(dev);
+ LASSERTF(rrc == GNI_RC_SUCCESS,
+ "bad rc from deregistion of sMDD buffer: %d\n", rrc);
+ dev->gnd_smdd_hold_buf = NULL;
+ }
+
if (dev->gnd_rcv_fma_cqh) {
rrc = kgnilnd_cq_destroy(dev->gnd_rcv_fma_cqh);
LASSERTF(rrc == GNI_RC_SUCCESS,
atomic_t gnd_n_schedule;
atomic_t gnd_canceled_dgrams; /* # of outstanding cancels */
struct rw_semaphore gnd_conn_sem; /* serialize connection changes/data movement */
+ void *gnd_smdd_hold_buf; /* buffer to keep smdd */
+ gni_mem_handle_t gnd_smdd_hold_hndl; /* buffer mem handle */
} kgn_device_t;
typedef struct kgn_net {
/* pulls in tunables per platform and adds in nid/nic conversion
* if RCA wasn't available at build time */
#include "gnilnd_hss_ops.h"
+/* API wrapper functions - include late to pick up all of the other defines */
+#include "gnilnd_api_wrap.h"
#if defined(CONFIG_CRAY_GEMINI)
#include "gnilnd_gemini.h"
#error "Undefined Network Hardware Type"
#endif
-/* API wrapper functions - include late to pick up all of the other defines */
-#include "gnilnd_api_wrap.h"
+extern uint32_t kgni_driver_version;
+
+static inline void
+kgnilnd_check_kgni_version(void)
+{
+ uint32_t *kdv;
+
+ kgnilnd_data.kgn_enable_gl_mutex = 1;
+ kdv = symbol_get(kgni_driver_version);
+ if (!kdv) {
+ LCONSOLE_INFO("Not using thread safe locking -"
+ " no symbol kgni_driver_version\n");
+ return;
+ }
+
+ /* Thread-safe kgni implemented in minor ver 0x44/45, code rev 0xb9 */
+ if (*kdv < GNI_VERSION_CHECK(0, GNILND_KGNI_TS_MINOR_VER, 0xb9)) {
+ symbol_put(kgni_driver_version);
+ LCONSOLE_INFO("Not using thread safe locking, gni version 0x%x,"
+ " need >= 0x%x\n", *kdv,
+ GNI_VERSION_CHECK(0, GNILND_KGNI_TS_MINOR_VER, 0xb9));
+ return;
+ }
+
+ symbol_put(kgni_driver_version);
+
+ if (!*kgnilnd_tunables.kgn_thread_safe) {
+ return;
+ }
+
+ /* Use thread-safe locking */
+ kgnilnd_data.kgn_enable_gl_mutex = 0;
+}
#endif /* _GNILND_GNILND_H_ */
#define CFS_FAIL_GNI_DGRAM_DEADLINE 0xf053
#define CFS_FAIL_GNI_DGRAM_DROP_TX 0xf054
-extern uint32_t kgni_driver_version;
-
-static inline void
-kgnilnd_check_kgni_version(void)
-{
- uint32_t *kdv;
-
- kgnilnd_data.kgn_enable_gl_mutex = 1;
- kdv = symbol_get(kgni_driver_version);
- if (!kdv) {
- LCONSOLE_INFO("Not using thread safe locking -"
- " no symbol kgni_driver_version\n");
- return;
- }
-
- /* Thread-safe kgni implemented in minor ver 0x44/45, code rev 0xb9 */
- if (*kdv < GNI_VERSION_CHECK(0, GNILND_KGNI_TS_MINOR_VER, 0xb9)) {
- symbol_put(kgni_driver_version);
- LCONSOLE_INFO("Not using thread safe locking, gni version 0x%x,"
- " need >= 0x%x\n", *kdv,
- GNI_VERSION_CHECK(0, GNILND_KGNI_TS_MINOR_VER, 0xb9));
- return;
- }
-
- symbol_put(kgni_driver_version);
-
- if (!*kgnilnd_tunables.kgn_thread_safe) {
- return;
- }
-
- /* Use thread-safe locking */
- kgnilnd_data.kgn_enable_gl_mutex = 0;
-}
-
/* helper macros */
extern void
_kgnilnd_api_rc_lbug(const char *rcstr, int rc, struct libcfs_debug_msg_data *data,
#define GNILND_KGNI_TS_MINOR_VER 0x45
#define GNILND_TS_ENABLE 1
+/* register some memory to allocate a shared mdd */
+static inline gni_return_t
+kgnilnd_register_smdd_buf(kgn_device_t *dev)
+{
+ __u32 flags = GNI_MEM_READWRITE;
+
+ if (*kgnilnd_tunables.kgn_bte_relaxed_ordering) {
+ flags |= GNI_MEM_RELAXED_PI_ORDERING;
+ }
+
+ LIBCFS_ALLOC(dev->gnd_smdd_hold_buf, PAGE_SIZE);
+ if (!dev->gnd_smdd_hold_buf) {
+ CERROR("Can't allocate smdd hold buffer\n");
+ return GNI_RC_ERROR_RESOURCE;
+ }
+
+ return kgnilnd_mem_register(dev->gnd_handle,
+ (__u64)dev->gnd_smdd_hold_buf,
+ PAGE_SIZE, NULL, flags,
+ &dev->gnd_smdd_hold_hndl);
+}
+
+static inline gni_return_t
+kgnilnd_deregister_smdd_buf(kgn_device_t *dev)
+{
+ gni_return_t rc = kgnilnd_mem_deregister(dev->gnd_handle,
+ &dev->gnd_smdd_hold_hndl, 0);
+ LIBCFS_FREE(dev->gnd_smdd_hold_buf, PAGE_SIZE);
+
+ return rc;
+}
+
/* plug in our functions for use on the simulator */
#if !defined(GNILND_USE_RCA)
#define GNILND_KGNI_TS_MINOR_VER 0x44
#define GNILND_TS_ENABLE 0
+static inline gni_return_t
+kgnilnd_register_smdd_buf(kgn_device_t *dev)
+{
+ return GNI_RC_SUCCESS;
+}
+
+static inline gni_return_t
+kgnilnd_deregister_smdd_buf(kgn_device_t *dev)
+{
+ return GNI_RC_SUCCESS;
+}
+
#endif /* _GNILND_GEMINI_H */