*
* Copyright (C) 2009-2012 Cray, Inc.
*
+ * Copyright (c) 2014, Intel Corporation.
+ *
* Derived from work by: Eric Barton <eric@bartonsoftware.com>
* Author: Nic Henke <nic@cray.com>
* Author: James Shimek <jshimek@cray.com>
#ifndef _GNILND_GNILND_H_
#define _GNILND_GNILND_H_
+#ifdef HAVE_COMPAT_RDMA
+#include <linux/compat-2.6.h>
+#endif
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/time.h>
#include <asm/timex.h>
-#include <asm/system.h>
#include <asm/uaccess.h>
#include <asm/io.h>
#include <libcfs/libcfs.h>
#include <lnet/lnet.h>
#include <lnet/lib-lnet.h>
-#include <lnet/lnet-sysctl.h>
#include <gni_pub.h>
#include "gnilnd_version.h"
/* fixed constants */
#define GNILND_MAXDEVS 1 /* max # of GNI devices currently supported */
#define GNILND_MBOX_CREDITS 256 /* number of credits per mailbox */
-#define GNILND_COOKIE 0xa3579 /* cookie used by along with ptag by GNI */
#define GNILND_CONN_MAGIC 0xa100f /* magic value for verifying connection validity */
/* checksum values */
#define GNILND_CHECKSUM_OFF 0 /* checksum turned off */
/* tune down some COMPUTE options as they won't see the same number of connections and
* don't need the throughput of multiple threads by default */
#if defined(CONFIG_CRAY_COMPUTE)
+#ifdef CONFIG_MK1OM
+#define GNILND_SCHED_THREADS 2 /* default # of kgnilnd_scheduler threads */
+#else
#define GNILND_SCHED_THREADS 1 /* default # of kgnilnd_scheduler threads */
+#endif
#define GNILND_FMABLK 64 /* default number of mboxes per fmablk */
#define GNILND_SCHED_NICE 0 /* default nice value for scheduler threads */
#define GNILND_COMPUTE 1 /* compute image */
+#define GNILND_FAST_RECONNECT 1 /* Fast Reconnect option */
#else
-#define GNILND_SCHED_THREADS 3 /* default # of kgnilnd_scheduler threads */
#define GNILND_FMABLK 1024 /* default number of mboxes per fmablk */
#define GNILND_SCHED_NICE -20 /* default nice value for scheduler threads */
#define GNILND_COMPUTE 0 /* service image */
+#define GNILND_FAST_RECONNECT 0 /* Fast Reconnect option */
#endif
/* EXTRA_BITS are there to allow us to hide NOOP/CLOSE and anything else out of band */
/* need sane upper bound to limit copy overhead */
#define GNILND_MAX_IMMEDIATE (64<<10)
+/* Max number of connections to keep in purgatory per peer */
+#define GNILND_PURGATORY_MAX 5
+/* Closing, don't put in purgatory */
+#define GNILND_NOPURG 222
+
/* payload size to add to the base mailbox size
* This is subtracting 2 from the concurrent_sends as 4 messages are included in the size
* gni_smsg_buff_size_needed calculates, the MAX_PAYLOAD is added to
int *kgn_bte_dlvr_mode; /* BTE delivery mode mask */
int *kgn_bte_relaxed_ordering; /* relaxed ordering (PASSPW) on BTE transfers */
int *kgn_ptag; /* PTAG for cdm_create */
+ int *kgn_pkey; /* PKEY for cdm_create */
int *kgn_max_retransmits; /* max number of FMA retransmits */
int *kgn_nwildcard; /* # wildcard per net to post */
int *kgn_nice; /* nice value for kgnilnd threads */
int *kgn_dgram_timeout; /* max time for dgram mover to run before scheduling */
int *kgn_sched_nice; /* nice value for kgnilnd scheduler threads */
int *kgn_reverse_rdma; /* Reverse RDMA setting */
+ int *kgn_eager_credits; /* allocated eager buffers */
+ int *kgn_fast_reconn; /* fast reconnection on conn timeout */
+ int *kgn_efault_lbug; /* LBUG on receiving an EFAULT */
+ int *kgn_max_purgatory; /* # conns/peer to keep in purgatory */
+ int *kgn_thread_affinity; /* bind scheduler threads to cpus */
+ int *kgn_thread_safe; /* use thread safe kgni API */
#if CONFIG_SYSCTL && !CFS_SYSFS_MODULE_PARM
- cfs_sysctl_table_header_t *kgn_sysctl; /* sysctl interface */
+ struct ctl_table_header *kgn_sysctl; /* sysctl interface */
#endif
} kgn_tunables_t;
int gnd_id; /* device id, also index in kgn_devices */
__u32 gnd_nid; /* ph host ID translated to NID */
struct list_head gnd_fma_buffs; /* list of FMA memory blocks */
- struct semaphore gnd_fmablk_sem; /* semaphore for FMA block memory alloc/free */
+ struct mutex gnd_fmablk_mutex; /* mutex for FMA block memory alloc/free */
spinlock_t gnd_fmablk_lock; /* lock for mbox alloc/release */
atomic_t gnd_nfmablk; /* # of fmablk live */
atomic_t gnd_fmablk_vers; /* gnd_fma_bufs stamp */
int gnd_dgram_ready; /* dgrams need movin' */
struct list_head *gnd_dgrams; /* nid hash to dgrams */
atomic_t gnd_ndgrams; /* # dgrams extant */
- atomic_t gnd_nwcdgrams; /* # wildcard dgrams to post on device */
+ atomic_t gnd_nwcdgrams; /* # wildcard dgrams to post*/
spinlock_t gnd_dgram_lock; /* serialize gnd_dgrams */
struct list_head gnd_map_list; /* list of all mapped regions */
int gnd_map_version; /* version flag for map list */
atomic_t gnd_n_schedule;
atomic_t gnd_canceled_dgrams; /* # of outstanding cancels */
struct rw_semaphore gnd_conn_sem; /* serialize connection changes/data movement */
+ void *gnd_smdd_hold_buf; /* buffer to keep smdd */
+ gni_mem_handle_t gnd_smdd_hold_hndl; /* buffer mem handle */
} kgn_device_t;
typedef struct kgn_net {
atomic_t gnc_sched_noop; /* # sched triggered NOOP */
unsigned int gnc_timeout; /* infer peer death if no rx for this many seconds */
__u32 gnc_cqid; /* my completion callback id (non-unique) */
- __u32 gnc_tx_seq; /* tx msg sequence number */
- __u32 gnc_rx_seq; /* rx msg sequence number */
+ atomic_t gnc_tx_seq; /* tx msg sequence number */
+ atomic_t gnc_rx_seq; /* rx msg sequence number */
+ struct mutex gnc_smsg_mutex; /* tx smsg sequence serialization */
+ struct mutex gnc_rdma_mutex; /* tx rdma sequence serialization */
__u64 gnc_tx_retrans; /* # retrans on SMSG */
atomic_t gnc_nlive_fma; /* # live FMA */
atomic_t gnc_nq_rdma; /* # queued (on device) RDMA */
wait_queue_head_t kgn_ruhroh_waitq; /* ruhroh thread wakeup */
int kgn_quiesce_trigger; /* should we quiesce ? */
atomic_t kgn_nquiesce; /* how many quiesced ? */
- struct semaphore kgn_quiesce_sem; /* serialize ruhroh task, startup and shutdown */
+ struct mutex kgn_quiesce_mutex; /* serialize ruhroh task, startup and shutdown */
int kgn_needs_reset; /* we need stack reset */
/* These next three members implement communication from gnilnd into
struct list_head *kgn_conns; /* conns hashed by cqid */
atomic_t kgn_nconns; /* # connections extant */
+ atomic_t kgn_neager_allocs; /* # of eager allocations */
__u64 kgn_peerstamp; /* when I started up */
__u64 kgn_connstamp; /* conn stamp generator */
int kgn_conn_version; /* version flag for conn tables */
wait_queue_head_t kgn_reaper_waitq; /* reaper sleeps here */
spinlock_t kgn_reaper_lock; /* serialise */
- struct kmem_cache *kgn_rx_cache; /* rx descriptor space */
- struct kmem_cache *kgn_tx_cache; /* tx descriptor memory */
- struct kmem_cache *kgn_tx_phys_cache; /* tx phys descriptor memory */
+ struct kmem_cache *kgn_rx_cache; /* rx descriptor space */
+ struct kmem_cache *kgn_tx_cache; /* tx descriptor memory */
+ struct kmem_cache *kgn_tx_phys_cache; /* tx phys descriptor memory */
atomic_t kgn_ntx; /* # tx in use */
- struct kmem_cache *kgn_dgram_cache; /* outgoing datagrams */
+ struct kmem_cache *kgn_dgram_cache; /* outgoing datagrams */
struct page ***kgn_cksum_map_pages; /* page arrays for mapping pages on checksum */
- __u64 kgn_cksum_npages; /* Number of pages allocated for checksumming */
+ __u64 kgn_cksum_npages; /* # pages alloc'd for checksumming */
atomic_t kgn_nvmap_cksum; /* # times we vmapped for checksums */
atomic_t kgn_nvmap_short; /* # times we vmapped for short kiov */
atomic_t kgn_npending_unlink; /* # of peers pending unlink */
atomic_t kgn_npending_conns; /* # of conns with pending closes */
atomic_t kgn_npending_detach; /* # of conns with a pending detach */
- unsigned long kgn_last_scheduled; /* last time schedule was called in a sched thread */
- unsigned long kgn_last_condresched; /* last time cond_resched was called in a sched thread */
- atomic_t kgn_rev_offset; /* number of time REV rdma have been misaligned offsets */
- atomic_t kgn_rev_length; /* Number of times REV rdma have been misaligned lengths */
- atomic_t kgn_rev_copy_buff; /* Number of times REV rdma have had to make a copy buffer */
+ unsigned long kgn_last_scheduled; /* last time schedule was called */
+ unsigned long kgn_last_condresched; /* last time cond_resched was called */
+ atomic_t kgn_rev_offset; /* # of REV rdma w/misaligned offsets */
+ atomic_t kgn_rev_length; /* # of REV rdma have misaligned len */
+ atomic_t kgn_rev_copy_buff; /* # of REV rdma buffer copies */
+ struct socket *kgn_sock; /* for Apollo */
+ unsigned long free_pages_limit; /* # of free pages reserve from fma block allocations */
+ int kgn_enable_gl_mutex; /* kgni api mtx enable */
} kgn_data_t;
extern kgn_data_t kgnilnd_data;
#define kgnilnd_schedule_conn(conn) \
_kgnilnd_schedule_conn(conn, __func__, __LINE__, 0);
-#define kgnilnd_schedule_conn_refheld(conn, refheld) \
+#define kgnilnd_schedule_conn_refheld(conn, refheld) \
_kgnilnd_schedule_conn(conn, __func__, __LINE__, refheld);
-static inline int
-kgnilnd_thread_start(int(*fn)(void *arg), void *arg, char *name, int id)
+static inline void
+kgnilnd_thread_fini(void)
{
- struct task_struct *thrd = kthread_run(fn, arg, "%s_%02d", name, id);
- if (IS_ERR(thrd))
- return PTR_ERR(thrd);
+ atomic_dec(&kgnilnd_data.kgn_nthreads);
+}
- atomic_inc(&kgnilnd_data.kgn_nthreads);
- return 0;
+static inline void kgnilnd_gl_mutex_lock(struct mutex *lock)
+{
+ if (kgnilnd_data.kgn_enable_gl_mutex)
+ mutex_lock(lock);
}
-static inline void
-kgnilnd_thread_fini(void)
+static inline void kgnilnd_gl_mutex_unlock(struct mutex *lock)
{
- atomic_dec(&kgnilnd_data.kgn_nthreads);
+ if (kgnilnd_data.kgn_enable_gl_mutex)
+ mutex_unlock(lock);
+}
+
+static inline void kgnilnd_conn_mutex_lock(struct mutex *lock)
+{
+ if (!kgnilnd_data.kgn_enable_gl_mutex)
+ mutex_lock(lock);
+}
+
+static inline void kgnilnd_conn_mutex_unlock(struct mutex *lock)
+{
+ if (!kgnilnd_data.kgn_enable_gl_mutex)
+ mutex_unlock(lock);
}
/* like mutex_trylock but with a jiffies spinner. This is to allow certain
* This function must not be used in interrupt context. The
* mutex must be released by the same task that acquired it.
*/
-static inline int kgnilnd_mutex_trylock(struct mutex *lock)
+static inline int __kgnilnd_mutex_trylock(struct mutex *lock)
{
int ret;
unsigned long timeout;
return 0;
}
+static inline int kgnilnd_mutex_trylock(struct mutex *lock)
+{
+ if (!kgnilnd_data.kgn_enable_gl_mutex)
+ return 1;
+
+ return __kgnilnd_mutex_trylock(lock);
+}
+
+static inline int kgnilnd_trylock(struct mutex *cq_lock,
+ struct mutex *c_lock)
+{
+ if (kgnilnd_data.kgn_enable_gl_mutex)
+ return __kgnilnd_mutex_trylock(cq_lock);
+ else
+ return __kgnilnd_mutex_trylock(c_lock);
+}
+
+static inline void *kgnilnd_vzalloc(int size)
+{
+ void *ret = __vmalloc(size, __GFP_HIGHMEM | GFP_NOFS | __GFP_ZERO,
+ PAGE_KERNEL);
+ LIBCFS_ALLOC_POST(ret, size);
+ return ret;
+}
+
/* Copied from DEBUG_REQ in Lustre - the dance is needed to save stack space */
extern void
atomic_inc(&kgnilnd_data.kgn_nquiesce); \
CDEBUG(D_NET, "Waiting for thread pause to be over...\n"); \
while (kgnilnd_data.kgn_quiesce_trigger) { \
- set_current_state(TASK_INTERRUPTIBLE); \
- schedule_timeout(HZ); \
+ msleep_interruptible(MSEC_PER_SEC); \
} \
/* Mom, my homework is done */ \
CDEBUG(D_NET, "Waking up from thread pause\n"); \
int kgnilnd_allocate_phys_fmablk(kgn_device_t *device);
int kgnilnd_map_phys_fmablk(kgn_device_t *device);
-void kgnilnd_unmap_phys_fmablk(kgn_device_t *device);
+void kgnilnd_unmap_fma_blocks(kgn_device_t *device);
void kgnilnd_free_phys_fmablk(kgn_device_t *device);
int kgnilnd_ctl(lnet_ni_t *ni, unsigned int cmd, void *arg);
lnet_msg_t *lntmsg, void **new_private);
int kgnilnd_recv(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg,
int delayed, unsigned int niov,
- struct iovec *iov, lnet_kiov_t *kiov,
+ struct kvec *iov, lnet_kiov_t *kiov,
unsigned int offset, unsigned int mlen, unsigned int rlen);
__u16 kgnilnd_cksum_kiov(unsigned int nkiov, lnet_kiov_t *kiov, unsigned int offset, unsigned int nob, int dump_blob);
int kgnilnd_schedule_process_conn(kgn_conn_t *conn, int sched_intent);
void kgnilnd_schedule_dgram(kgn_device_t *dev);
-int kgnilnd_create_peer_safe(kgn_peer_t **peerp, lnet_nid_t nid, kgn_net_t *net);
+int kgnilnd_create_peer_safe(kgn_peer_t **peerp, lnet_nid_t nid, kgn_net_t *net, int node_state);
void kgnilnd_add_peer_locked(lnet_nid_t nid, kgn_peer_t *new_stub_peer, kgn_peer_t **peerp);
int kgnilnd_add_peer(kgn_net_t *net, lnet_nid_t nid, kgn_peer_t **peerp);
int kgnilnd_scheduler(void *arg);
int kgnilnd_dgram_mover(void *arg);
int kgnilnd_rca(void *arg);
+int kgnilnd_thread_start(int(*fn)(void *arg), void *arg, char *name, int id);
int kgnilnd_create_conn(kgn_conn_t **connp, kgn_device_t *dev);
int kgnilnd_conn_isdup_locked(kgn_peer_t *peer, kgn_conn_t *newconn);
void kgnilnd_cancel_peer_connect_locked(kgn_peer_t *peer, struct list_head *zombies);
int kgnilnd_close_stale_conns_locked(kgn_peer_t *peer, kgn_conn_t *newconn);
void kgnilnd_peer_alive(kgn_peer_t *peer);
-void kgnilnd_peer_notify(kgn_peer_t *peer, int error);
+void kgnilnd_peer_notify(kgn_peer_t *peer, int error, int alive);
void kgnilnd_close_conn_locked(kgn_conn_t *conn, int error);
void kgnilnd_close_conn(kgn_conn_t *conn, int error);
void kgnilnd_complete_closed_conn(kgn_conn_t *conn);
int kgnilnd_report_node_state(lnet_nid_t nid, int down);
void kgnilnd_wakeup_rca_thread(void);
int kgnilnd_start_rca_thread(void);
+int kgnilnd_get_node_state(__u32 nid);
int kgnilnd_tunables_init(void);
void kgnilnd_tunables_fini(void);
int kgnilnd_find_and_cancel_dgram(kgn_device_t *dev, lnet_nid_t dst_nid);
void kgnilnd_cancel_dgram_locked(kgn_dgram_t *dgram);
-void kgnilnd_release_dgram(kgn_device_t *dev, kgn_dgram_t *dgram);
+void kgnilnd_release_dgram(kgn_device_t *dev, kgn_dgram_t *dgram, int shutdown);
int kgnilnd_setup_wildcard_dgram(kgn_device_t *dev);
int kgnilnd_cancel_net_dgrams(kgn_net_t *net);
int kgnilnd_cancel_wc_dgrams(kgn_device_t *dev);
+int kgnilnd_cancel_dgrams(kgn_device_t *dev);
void kgnilnd_wait_for_canceled_dgrams(kgn_device_t *dev);
int kgnilnd_dgram_waitq(void *arg);
#undef DO_TYPE
-/* API wrapper functions - include late to pick up all of the other defines */
-#include "gnilnd_api_wrap.h"
-
/* pulls in tunables per platform and adds in nid/nic conversion
* if RCA wasn't available at build time */
#include "gnilnd_hss_ops.h"
+/* API wrapper functions - include late to pick up all of the other defines */
+#include "gnilnd_api_wrap.h"
#if defined(CONFIG_CRAY_GEMINI)
#include "gnilnd_gemini.h"
#error "Undefined Network Hardware Type"
#endif
+extern uint32_t kgni_driver_version;
+
+static inline void
+kgnilnd_check_kgni_version(void)
+{
+ uint32_t *kdv;
+
+ kgnilnd_data.kgn_enable_gl_mutex = 1;
+ kdv = symbol_get(kgni_driver_version);
+ if (!kdv) {
+ LCONSOLE_INFO("Not using thread safe locking -"
+ " no symbol kgni_driver_version\n");
+ return;
+ }
+
+ /* Thread-safe kgni implemented in minor ver 0x44/45, code rev 0xb9 */
+ if (*kdv < GNI_VERSION_CHECK(0, GNILND_KGNI_TS_MINOR_VER, 0xb9)) {
+ symbol_put(kgni_driver_version);
+ LCONSOLE_INFO("Not using thread safe locking, gni version 0x%x,"
+ " need >= 0x%x\n", *kdv,
+ GNI_VERSION_CHECK(0, GNILND_KGNI_TS_MINOR_VER, 0xb9));
+ return;
+ }
+
+ symbol_put(kgni_driver_version);
+
+ if (!*kgnilnd_tunables.kgn_thread_safe) {
+ return;
+ }
+
+ /* Use thread-safe locking */
+ kgnilnd_data.kgn_enable_gl_mutex = 0;
+}
+
#endif /* _GNILND_GNILND_H_ */
git://git.whamcloud.com / fs / lustre-release.git / blobdiff