X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lnet%2Fklnds%2Fgmlnd%2Fgmlnd.h;h=5eb844dc37d900269dd19acb06fa2fcc94a35ca4;hb=b48ab0632ba0c88326c8d9466760bf56301b3676;hp=f45eab7744f13ee05d65688df225492a99501c72;hpb=3fed8a248a88129a08d2579a988249e648931582;p=fs%2Flustre-release.git diff --git a/lnet/klnds/gmlnd/gmlnd.h b/lnet/klnds/gmlnd/gmlnd.h index f45eab7..5eb844d 100644 --- a/lnet/klnds/gmlnd/gmlnd.h +++ b/lnet/klnds/gmlnd/gmlnd.h @@ -21,18 +21,26 @@ /* - * Portals GM kernel NAL header file - * This file makes all declaration and prototypes - * for the API side and CB side of the NAL + * Portals GM kernel NAL header file + * This file makes all declaration and prototypes + * for the API side and CB side of the NAL */ #ifndef __INCLUDE_GMNAL_H__ #define __INCLUDE_GMNAL_H__ +/* XXX Lustre as of V1.2.2 drop defines VERSION, which causes problems + * when including /include/gm_lanai.h which defines a structure field + * with the name VERSION XXX */ +#ifdef VERSION +# undef VERSION +#endif + #ifndef EXPORT_SYMTAB # define EXPORT_SYMTAB #endif - -#include "linux/config.h" +#ifndef AUTOCONF_INCLUDED +#include +#endif #include "linux/module.h" #include "linux/tty.h" #include "linux/kernel.h" @@ -40,23 +48,31 @@ #include "linux/string.h" #include "linux/stat.h" #include "linux/errno.h" +#include "linux/version.h" +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,0) +#include "linux/buffer_head.h" +#include "linux/fs.h" +#else #include "linux/locks.h" +#endif #include "linux/unistd.h" #include "linux/init.h" #include "linux/sem.h" #include "linux/vmalloc.h" #include "linux/sysctl.h" -#define DEBUG_SUBSYSTEM S_NAL +#define DEBUG_SUBSYSTEM S_LND -#include "portals/nal.h" -#include "portals/api.h" -#include "portals/errno.h" -#include "linux/kp30.h" -#include "portals/p30.h" +#include "libcfs/kp30.h" +#include "lnet/lnet.h" +#include "lnet/lib-lnet.h" -#include "portals/nal.h" -#include "portals/lib-p30.h" +/* undefine these before including the GM headers which clash */ +#undef PACKAGE_BUGREPORT +#undef PACKAGE_NAME +#undef PACKAGE_STRING +#undef PACKAGE_TARNAME +#undef PACKAGE_VERSION #define GM_STRONG_TYPES 1 #ifdef VERSION @@ -65,391 +81,166 @@ #include "gm.h" #include "gm_internal.h" +/* Fixed tunables */ +#define GMNAL_RESCHED 100 /* # busy loops to force scheduler to yield */ +#define GMNAL_NETADDR_BASE 0x10000000 /* where we start in network VM */ +#define GMNAL_LARGE_PRIORITY GM_LOW_PRIORITY /* large message GM priority */ +#define GMNAL_SMALL_PRIORITY GM_LOW_PRIORITY /* small message GM priority */ + +/* Wire protocol */ +typedef struct { + lnet_hdr_t gmim_hdr; /* portals header */ + char gmim_payload[0]; /* payload */ +} gmnal_immediate_msg_t; + +typedef struct { + /* First 2 fields fixed FOR ALL TIME */ + __u32 gmm_magic; /* I'm a GM message */ + __u16 gmm_version; /* this is my version number */ + + __u16 gmm_type; /* msg type */ + __u64 gmm_srcnid; /* sender's NID */ + __u64 gmm_dstnid; /* destination's NID */ + union { + gmnal_immediate_msg_t immediate; + } gmm_u; +} WIRE_ATTR gmnal_msg_t; + +#define GMNAL_MSG_MAGIC LNET_PROTO_GM_MAGIC +#define GMNAL_MSG_VERSION 1 +#define GMNAL_MSG_IMMEDIATE 1 + +typedef struct netbuf { + __u64 nb_netaddr; /* network VM address */ + lnet_kiov_t nb_kiov[1]; /* the pages (at least 1) */ +} gmnal_netbuf_t; + +#define GMNAL_NETBUF_MSG(nb) ((gmnal_msg_t *)page_address((nb)->nb_kiov[0].kiov_page)) +#define GMNAL_NETBUF_LOCAL_NETADDR(nb) ((void *)((unsigned long)(nb)->nb_netaddr)) + +typedef struct gmnal_txbuf { + struct list_head txb_list; /* queue on gmni_idle_ltxbs */ + struct gmnal_txbuf *txb_next; /* stash on gmni_ltxs */ + gmnal_netbuf_t txb_buf; /* space */ +} gmnal_txbuf_t; + +typedef struct gmnal_tx { + struct list_head tx_list; /* queue */ + int tx_credit:1; /* consumed a credit? */ + int tx_large_iskiov:1; /* large is in kiovs? */ + struct gmnal_ni *tx_gmni; /* owning NI */ + lnet_nid_t tx_nid; /* destination NID */ + int tx_gmlid; /* destination GM local ID */ + lnet_msg_t *tx_lntmsg; /* lntmsg to finalize on completion */ + + gmnal_netbuf_t tx_buf; /* small tx buffer */ + gmnal_txbuf_t *tx_ltxb; /* large buffer (to free on completion) */ + int tx_msgnob; /* message size (so far) */ + + int tx_large_nob; /* # bytes large buffer payload */ + int tx_large_offset; /* offset within frags */ + int tx_large_niov; /* # VM frags */ + union { + struct iovec *iov; /* mapped frags */ + lnet_kiov_t *kiov; /* page frags */ + } tx_large_frags; + unsigned long tx_launchtime; /* when (in jiffies) the transmit was launched */ + struct gmnal_tx *tx_next; /* stash on gmni_txs */ +} gmnal_tx_t; + +typedef struct gmnal_rx { + struct list_head rx_list; /* enqueue on gmni_rxq for handling */ + int rx_islarge:1; /* large receive buffer? */ + unsigned int rx_recv_nob; /* bytes received */ + __u16 rx_recv_gmid; /* sender */ + __u8 rx_recv_port; /* sender's port */ + __u8 rx_recv_type; /* ?? */ + struct gmnal_rx *rx_next; /* stash on gmni_rxs */ + gmnal_netbuf_t rx_buf; /* the buffer */ +} gmnal_rx_t; + +typedef struct gmnal_ni { + lnet_ni_t *gmni_ni; /* generic NI */ + struct gm_port *gmni_port; /* GM port */ + spinlock_t gmni_gm_lock; /* serialise GM calls */ + int gmni_large_pages; /* # pages in a large message buffer */ + int gmni_large_msgsize; /* nob in large message buffers */ + int gmni_large_gmsize; /* large message GM bucket */ + int gmni_small_msgsize; /* nob in small message buffers */ + int gmni_small_gmsize; /* small message GM bucket */ + __u64 gmni_netaddr_base; /* base of mapped network VM */ + int gmni_netaddr_size; /* # bytes of mapped network VM */ + + gmnal_tx_t *gmni_txs; /* all txs */ + gmnal_rx_t *gmni_rxs; /* all rx descs */ + gmnal_txbuf_t *gmni_ltxbs; /* all large tx bufs */ + + atomic_t gmni_nthreads; /* total # threads */ + gm_alarm_t gmni_alarm; /* alarm to wake caretaker */ + int gmni_shutdown; /* tell all threads to exit */ + + struct list_head gmni_idle_txs; /* idle tx's */ + int gmni_tx_credits; /* # transmits still possible */ + struct list_head gmni_idle_ltxbs; /* idle large tx buffers */ + struct list_head gmni_buf_txq; /* tx's waiting for buffers */ + struct list_head gmni_cred_txq; /* tx's waiting for credits */ + spinlock_t gmni_tx_lock; /* serialise */ + + struct gm_hash *gmni_rx_hash; /* buffer->rx lookup */ + struct semaphore gmni_rx_mutex; /* serialise blocking on GM */ +} gmnal_ni_t; + +typedef struct { + int *gm_port; + int *gm_ntx; + int *gm_credits; + int *gm_peer_credits; + int *gm_nlarge_tx_bufs; + int *gm_nrx_small; + int *gm_nrx_large; + +#if CONFIG_SYSCTL && !CFS_SYSFS_MODULE_PARM + struct ctl_table_header *gm_sysctl; /* sysctl interface */ +#endif +} gmnal_tunables_t; -/* - * Defines for the API NAL - */ - -/* - * Small message size is configurable - * insmod can set small_msg_size - * which is used to populate nal_data.small_msg_size - */ -#define GMNAL_SMALL_MESSAGE 1078 -#define GMNAL_LARGE_MESSAGE_INIT 1079 -#define GMNAL_LARGE_MESSAGE_ACK 1080 -#define GMNAL_LARGE_MESSAGE_FINI 1081 - -extern int gmnal_small_msg_size; -extern int num_rx_threads; -extern int num_stxds; -extern int gm_port; -#define GMNAL_SMALL_MSG_SIZE(a) a->small_msg_size -#define GMNAL_IS_SMALL_MESSAGE(n,a,b,c) gmnal_is_small_msg(n, a, b, c) -#define GMNAL_MAGIC 0x1234abcd -/* - * The gm_port to use for gmnal - */ -#define GMNAL_GM_PORT gm_port - - -/* - * Small Transmit Descriptor - * A structre to keep track of a small transmit operation - * This structure has a one-to-one relationship with a small - * transmit buffer (both create by gmnal_stxd_alloc). - * There are two free list of stxd. One for use by clients of the NAL - * and the other by the NAL rxthreads when doing sends. - * This helps prevent deadlock caused by stxd starvation. - */ -typedef struct _gmnal_stxd_t { - void *buffer; - int buffer_size; - gm_size_t gm_size; - int msg_size; - int gm_target_node; - int gm_priority; - int type; - struct _gmnal_data_t *nal_data; - lib_msg_t *cookie; - int niov; - struct iovec iov[PTL_MD_MAX_IOV]; - struct _gmnal_stxd_t *next; - int rxt; - int kniov; - struct iovec *iovec_dup; -} gmnal_stxd_t; - -/* - * keeps a transmit token for large transmit (gm_get) - * and a pointer to rxd that is used as context for large receive - */ -typedef struct _gmnal_ltxd_t { - struct _gmnal_ltxd_t *next; - struct _gmnal_srxd_t *srxd; -} gmnal_ltxd_t; - - -/* - * as for gmnal_stxd_t - * a hash table in nal_data find srxds from - * the rx buffer address. hash table populated at init time - */ -typedef struct _gmnal_srxd_t { - void *buffer; - int size; - gm_size_t gmsize; - unsigned int gm_source_node; - gmnal_stxd_t *source_stxd; - int type; - int nsiov; - int nriov; - struct iovec *riov; - int ncallbacks; - spinlock_t callback_lock; - int callback_status; - lib_msg_t *cookie; - struct _gmnal_srxd_t *next; - struct _gmnal_data_t *nal_data; -} gmnal_srxd_t; - -/* - * Header which lmgnal puts at the start of each message - */ -typedef struct _gmnal_msghdr { - int magic; - int type; - unsigned int sender_node_id; - gmnal_stxd_t *stxd; - int niov; - } gmnal_msghdr_t; -#define GMNAL_MSGHDR_SIZE sizeof(gmnal_msghdr_t) - -/* - * the caretaker thread (ct_thread) gets receive events - * (and other events) from the myrinet device via the GM2 API. - * caretaker thread populates one work entry for each receive event, - * puts it on a Q in nal_data and wakes a receive thread to - * process the receive. - * Processing a portals receive can involve a transmit operation. - * Because of this the caretaker thread cannot process receives - * as it may get deadlocked when supply of transmit descriptors - * is exhausted (as caretaker thread is responsible for replacing - * transmit descriptors on the free list) - */ -typedef struct _gmnal_rxtwe { - void *buffer; - unsigned snode; - unsigned sport; - unsigned type; - unsigned length; - struct _gmnal_rxtwe *next; -} gmnal_rxtwe_t; - -/* - * 1 receive thread started on each CPU - */ -#define NRXTHREADS 10 /* max number of receiver threads */ - -typedef struct _gmnal_data_t { - spinlock_t stxd_lock; - struct semaphore stxd_token; - gmnal_stxd_t *stxd; - spinlock_t rxt_stxd_lock; - struct semaphore rxt_stxd_token; - gmnal_stxd_t *rxt_stxd; - spinlock_t ltxd_lock; - struct semaphore ltxd_token; - gmnal_ltxd_t *ltxd; - spinlock_t srxd_lock; - struct semaphore srxd_token; - gmnal_srxd_t *srxd; - struct gm_hash *srxd_hash; - nal_t *nal; - lib_nal_t *libnal; - struct gm_port *gm_port; - unsigned int gm_local_nid; - unsigned int gm_global_nid; - spinlock_t gm_lock; - long rxthread_pid[NRXTHREADS]; - int rxthread_stop_flag; - spinlock_t rxthread_flag_lock; - long rxthread_flag; - long ctthread_pid; - int ctthread_flag; - gm_alarm_t ctthread_alarm; - int small_msg_size; - int small_msg_gmsize; - gmnal_rxtwe_t *rxtwe_head; - gmnal_rxtwe_t *rxtwe_tail; - spinlock_t rxtwe_lock; - struct semaphore rxtwe_wait; - struct ctl_table_header *sysctl; -} gmnal_data_t; - -/* - * Flags to start/stop and check status of threads - * each rxthread sets 1 bit (any bit) of the flag on startup - * and clears 1 bit when exiting - */ -#define GMNAL_THREAD_RESET 0 -#define GMNAL_THREAD_STOP 666 -#define GMNAL_CTTHREAD_STARTED 333 -#define GMNAL_RXTHREADS_STARTED ( (1<stxd_lock); -#define GMNAL_TXD_LOCK(a) spin_lock(&a->stxd_lock); -#define GMNAL_TXD_UNLOCK(a) spin_unlock(&a->stxd_lock); -#define GMNAL_TXD_TOKEN_INIT(a, n) sema_init(&a->stxd_token, n); -#define GMNAL_TXD_GETTOKEN(a) down(&a->stxd_token); -#define GMNAL_TXD_TRYGETTOKEN(a) down_trylock(&a->stxd_token) -#define GMNAL_TXD_RETURNTOKEN(a) up(&a->stxd_token); - -#define GMNAL_RXT_TXD_LOCK_INIT(a) spin_lock_init(&a->rxt_stxd_lock); -#define GMNAL_RXT_TXD_LOCK(a) spin_lock(&a->rxt_stxd_lock); -#define GMNAL_RXT_TXD_UNLOCK(a) spin_unlock(&a->rxt_stxd_lock); -#define GMNAL_RXT_TXD_TOKEN_INIT(a, n) sema_init(&a->rxt_stxd_token, n); -#define GMNAL_RXT_TXD_GETTOKEN(a) down(&a->rxt_stxd_token); -#define GMNAL_RXT_TXD_TRYGETTOKEN(a) down_trylock(&a->rxt_stxd_token) -#define GMNAL_RXT_TXD_RETURNTOKEN(a) up(&a->rxt_stxd_token); - -#define GMNAL_LTXD_LOCK_INIT(a) spin_lock_init(&a->ltxd_lock); -#define GMNAL_LTXD_LOCK(a) spin_lock(&a->ltxd_lock); -#define GMNAL_LTXD_UNLOCK(a) spin_unlock(&a->ltxd_lock); -#define GMNAL_LTXD_TOKEN_INIT(a, n) sema_init(&a->ltxd_token, n); -#define GMNAL_LTXD_GETTOKEN(a) down(&a->ltxd_token); -#define GMNAL_LTXD_TRYGETTOKEN(a) down_trylock(&a->ltxd_token) -#define GMNAL_LTXD_RETURNTOKEN(a) up(&a->ltxd_token); - -#define GMNAL_RXD_LOCK_INIT(a) spin_lock_init(&a->srxd_lock); -#define GMNAL_RXD_LOCK(a) spin_lock(&a->srxd_lock); -#define GMNAL_RXD_UNLOCK(a) spin_unlock(&a->srxd_lock); -#define GMNAL_RXD_TOKEN_INIT(a, n) sema_init(&a->srxd_token, n); -#define GMNAL_RXD_GETTOKEN(a) down(&a->srxd_token); -#define GMNAL_RXD_TRYGETTOKEN(a) down_trylock(&a->srxd_token) -#define GMNAL_RXD_RETURNTOKEN(a) up(&a->srxd_token); - -#define GMNAL_GM_LOCK_INIT(a) spin_lock_init(&a->gm_lock); -#define GMNAL_GM_LOCK(a) spin_lock(&a->gm_lock); -#define GMNAL_GM_UNLOCK(a) spin_unlock(&a->gm_lock); - - -/* - * Memory Allocator - */ - -/* - * API NAL - */ -int gmnal_api_startup(nal_t *, ptl_pid_t, - ptl_ni_limits_t *, ptl_ni_limits_t *); - -int gmnal_api_forward(nal_t *, int, void *, size_t, void *, size_t); - -void gmnal_api_shutdown(nal_t *); - -int gmnal_api_validate(nal_t *, void *, size_t); - -void gmnal_api_yield(nal_t *, unsigned long *, int); - -void gmnal_api_lock(nal_t *, unsigned long *); - -void gmnal_api_unlock(nal_t *, unsigned long *); - - -#define GMNAL_INIT_NAL(a) do { \ - (a)->nal_ni_init = gmnal_api_startup; \ - (a)->nal_ni_fini = gmnal_api_shutdown; \ - (a)->nal_data = NULL; \ - } while (0) - - -/* - * CB NAL - */ - -ptl_err_t gmnal_cb_send(lib_nal_t *, void *, lib_msg_t *, ptl_hdr_t *, - int, ptl_nid_t, ptl_pid_t, unsigned int, struct iovec *, size_t, size_t); - -ptl_err_t gmnal_cb_send_pages(lib_nal_t *, void *, lib_msg_t *, ptl_hdr_t *, - int, ptl_nid_t, ptl_pid_t, unsigned int, ptl_kiov_t *, size_t, size_t); - -ptl_err_t gmnal_cb_recv(lib_nal_t *, void *, lib_msg_t *, - unsigned int, struct iovec *, size_t, size_t, size_t); - -ptl_err_t gmnal_cb_recv_pages(lib_nal_t *, void *, lib_msg_t *, - unsigned int, ptl_kiov_t *, size_t, size_t, size_t); - -int gmnal_cb_dist(lib_nal_t *, ptl_nid_t, unsigned long *); - +/* gmnal_api.c */ int gmnal_init(void); - -void gmnal_fini(void); - - - -#define GMNAL_INIT_NAL_CB(a) do { \ - a->libnal_send = gmnal_cb_send; \ - a->libnal_send_pages = gmnal_cb_send_pages; \ - a->libnal_recv = gmnal_cb_recv; \ - a->libnal_recv_pages = gmnal_cb_recv_pages; \ - a->libnal_map = NULL; \ - a->libnal_unmap = NULL; \ - a->libnal_dist = gmnal_cb_dist; \ - a->libnal_data = NULL; \ - } while (0) - - -/* - * Small and Large Transmit and Receive Descriptor Functions - */ -int gmnal_alloc_txd(gmnal_data_t *); -void gmnal_free_txd(gmnal_data_t *); -gmnal_stxd_t* gmnal_get_stxd(gmnal_data_t *, int); -void gmnal_return_stxd(gmnal_data_t *, gmnal_stxd_t *); -gmnal_ltxd_t* gmnal_get_ltxd(gmnal_data_t *); -void gmnal_return_ltxd(gmnal_data_t *, gmnal_ltxd_t *); - -int gmnal_alloc_srxd(gmnal_data_t *); -void gmnal_free_srxd(gmnal_data_t *); -gmnal_srxd_t* gmnal_get_srxd(gmnal_data_t *, int); -void gmnal_return_srxd(gmnal_data_t *, gmnal_srxd_t *); - -/* - * general utility functions - */ -gmnal_srxd_t *gmnal_rxbuffer_to_srxd(gmnal_data_t *, void*); -void gmnal_stop_rxthread(gmnal_data_t *); -void gmnal_stop_ctthread(gmnal_data_t *); -void gmnal_small_tx_callback(gm_port_t *, void *, gm_status_t); -void gmnal_drop_sends_callback(gm_port_t *, void *, gm_status_t); -void gmnal_resume_sending_callback(gm_port_t *, void *, gm_status_t); -char *gmnal_gm_error(gm_status_t); -char *gmnal_rxevent(gm_recv_event_t*); -int gmnal_is_small_msg(gmnal_data_t*, int, struct iovec*, int); -void gmnal_yield(int); -int gmnal_start_kernel_threads(gmnal_data_t *); - - -/* - * Communication functions - */ - -/* - * Receive threads - */ -int gmnal_ct_thread(void *); /* caretaker thread */ -int gmnal_rx_thread(void *); /* receive thread */ -int gmnal_pre_receive(gmnal_data_t*, gmnal_rxtwe_t*, int); -int gmnal_rx_bad(gmnal_data_t *, gmnal_rxtwe_t *, gmnal_srxd_t*); -int gmnal_rx_requeue_buffer(gmnal_data_t *, gmnal_srxd_t *); -int gmnal_add_rxtwe(gmnal_data_t *, gm_recv_t *); -gmnal_rxtwe_t * gmnal_get_rxtwe(gmnal_data_t *); -void gmnal_remove_rxtwe(gmnal_data_t *); - - -/* - * Small messages - */ -int gmnal_small_rx(lib_nal_t *, void *, lib_msg_t *, unsigned int, - struct iovec *, size_t, size_t, size_t); -int gmnal_small_tx(lib_nal_t *, void *, lib_msg_t *, ptl_hdr_t *, - int, ptl_nid_t, ptl_pid_t, - unsigned int, struct iovec*, size_t, int); -void gmnal_small_tx_callback(gm_port_t *, void *, gm_status_t); - - - -/* - * Large messages - */ -int gmnal_large_rx(lib_nal_t *, void *, lib_msg_t *, unsigned int, - struct iovec *, size_t, size_t, size_t); - -int gmnal_large_tx(lib_nal_t *, void *, lib_msg_t *, ptl_hdr_t *, - int, ptl_nid_t, ptl_pid_t, unsigned int, - struct iovec*, size_t, int); - -void gmnal_large_tx_callback(gm_port_t *, void *, gm_status_t); - -int gmnal_remote_get(gmnal_srxd_t *, int, struct iovec*, int, - struct iovec*); - -void gmnal_remote_get_callback(gm_port_t *, void *, gm_status_t); - -int gmnal_copyiov(int, gmnal_srxd_t *, int, struct iovec*, int, - struct iovec*); - -void gmnal_large_tx_ack(gmnal_data_t *, gmnal_srxd_t *); -void gmnal_large_tx_ack_callback(gm_port_t *, void *, gm_status_t); -void gmnal_large_tx_ack_received(gmnal_data_t *, gmnal_srxd_t *); +void gmnal_fini(void); +int gmnal_ctl(lnet_ni_t *ni, unsigned int cmd, void *arg); +int gmnal_startup(lnet_ni_t *ni); +void gmnal_shutdown(lnet_ni_t *ni); + +/* gmnal_cb.c */ +int gmnal_recv(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg, + int delayed, unsigned int niov, + struct iovec *iov, lnet_kiov_t *kiov, + unsigned int offset, unsigned int mlen, unsigned int rlen); +int gmnal_send(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg); + +/* gmnal_util.c */ +void gmnal_free_ltxbufs(gmnal_ni_t *gmni); +int gmnal_alloc_ltxbufs(gmnal_ni_t *gmni); +void gmnal_free_txs(gmnal_ni_t *gmni); +int gmnal_alloc_txs(gmnal_ni_t *gmni); +void gmnal_free_rxs(gmnal_ni_t *gmni); +int gmnal_alloc_rxs(gmnal_ni_t *gmni); +char *gmnal_gmstatus2str(gm_status_t status); +char *gmnal_rxevent2str(gm_recv_event_t *ev); +void gmnal_yield(int delay); + +/* gmnal_comm.c */ +void gmnal_post_rx(gmnal_ni_t *gmni, gmnal_rx_t *rx); +gmnal_tx_t *gmnal_get_tx(gmnal_ni_t *gmni); +void gmnal_tx_done(gmnal_tx_t *tx, int rc); +void gmnal_pack_msg(gmnal_ni_t *gmni, gmnal_msg_t *msg, + lnet_nid_t dstnid, int type); +void gmnal_stop_threads(gmnal_ni_t *gmni); +int gmnal_start_threads(gmnal_ni_t *gmni); +void gmnal_check_txqueues_locked (gmnal_ni_t *gmni); + +/* Module Parameters */ +extern gmnal_tunables_t gmnal_tunables; #endif /*__INCLUDE_GMNAL_H__*/