Whamcloud - gitweb
Land b1_2 onto HEAD (20040317_2319)
[fs/lustre-release.git] / lustre / portals / include / linux / kp30.h
index 85fe8e7..c080a57 100644 (file)
@@ -7,10 +7,10 @@
 #define PORTAL_DEBUG
 
 #ifndef offsetof
-# define offsetof(typ,memb)    ((int)((char *)&(((typ *)0)->memb)))
+# define offsetof(typ,memb)     ((int)((char *)&(((typ *)0)->memb)))
 #endif
 
-#define LOWEST_BIT_SET(x)      ((x) & ~((x) - 1))
+#define LOWEST_BIT_SET(x)       ((x) & ~((x) - 1))
 
 /*
  *  Debugging
@@ -19,63 +19,70 @@ extern unsigned int portal_subsystem_debug;
 extern unsigned int portal_stack;
 extern unsigned int portal_debug;
 extern unsigned int portal_printk;
+extern unsigned int portal_cerror;
 /* Debugging subsystems (32 bits, non-overlapping) */
-#define S_UNDEFINED    (1 << 0)
-#define S_MDC          (1 << 1)
-#define S_MDS          (1 << 2)
-#define S_OSC          (1 << 3)
-#define S_OST          (1 << 4)
-#define S_CLASS        (1 << 5)
-#define S_LOG          (1 << 6)
-#define S_LLITE        (1 << 7)
-#define S_RPC          (1 << 8)
-#define S_MGMT         (1 << 9)
-#define S_PORTALS     (1 << 10)
-#define S_SOCKNAL     (1 << 11)
-#define S_QSWNAL      (1 << 12)
-#define S_PINGER      (1 << 13)
-#define S_FILTER      (1 << 14)
-#define S_PTLBD       (1 << 15)
-#define S_ECHO        (1 << 16)
-#define S_LDLM        (1 << 17)
-#define S_LOV         (1 << 18)
-#define S_GMNAL       (1 << 19)
-#define S_PTLROUTER   (1 << 20)
-#define S_COBD        (1 << 21)
+#define S_UNDEFINED   0x00000001
+#define S_MDC         0x00000002
+#define S_MDS         0x00000004
+#define S_OSC         0x00000008
+#define S_OST         0x00000010
+#define S_CLASS       0x00000020
+#define S_LOG         0x00000040
+#define S_LLITE       0x00000080
+#define S_RPC         0x00000100
+#define S_MGMT        0x00000200
+#define S_PORTALS     0x00000400
+#define S_SOCKNAL     0x00000800
+#define S_QSWNAL      0x00001000
+#define S_PINGER      0x00002000
+#define S_FILTER      0x00004000
+#define S_PTLBD       0x00008000
+#define S_ECHO        0x00010000
+#define S_LDLM        0x00020000
+#define S_LOV         0x00040000
+#define S_GMNAL       0x00080000
+#define S_PTLROUTER   0x00100000
+#define S_COBD        0x00200000
+#define S_IBNAL       0x00400000
 
 /* If you change these values, please keep portals/utils/debug.c
  * up to date! */
 
 /* Debugging masks (32 bits, non-overlapping) */
-#define D_TRACE     (1 << 0) /* ENTRY/EXIT markers */
-#define D_INODE     (1 << 1)
-#define D_SUPER     (1 << 2)
-#define D_EXT2      (1 << 3) /* anything from ext2_debug */
-#define D_MALLOC    (1 << 4) /* print malloc, free information */
-#define D_CACHE     (1 << 5) /* cache-related items */
-#define D_INFO      (1 << 6) /* general information */
-#define D_IOCTL     (1 << 7) /* ioctl related information */
-#define D_BLOCKS    (1 << 8) /* ext2 block allocation */
-#define D_NET       (1 << 9) /* network communications */
-#define D_WARNING   (1 << 10)
-#define D_BUFFS     (1 << 11)
-#define D_OTHER     (1 << 12)
-#define D_DENTRY    (1 << 13)
-#define D_PORTALS   (1 << 14) /* ENTRY/EXIT markers */
-#define D_PAGE      (1 << 15) /* bulk page handling */
-#define D_DLMTRACE  (1 << 16)
-#define D_ERROR     (1 << 17) /* CERROR(...) == CDEBUG (D_ERROR, ...) */
-#define D_EMERG     (1 << 18) /* CEMERG(...) == CDEBUG (D_EMERG, ...) */
-#define D_HA        (1 << 19) /* recovery and failover */
-#define D_RPCTRACE  (1 << 20) /* for distributed debugging */
-#define D_VFSTRACE  (1 << 21)
+#define D_TRACE       0x00000001 /* ENTRY/EXIT markers */
+#define D_INODE       0x00000002
+#define D_SUPER       0x00000004
+#define D_EXT2        0x00000008 /* anything from ext2_debug */
+#define D_MALLOC      0x00000010 /* print malloc, free information */
+#define D_CACHE       0x00000020 /* cache-related items */
+#define D_INFO        0x00000040 /* general information */
+#define D_IOCTL       0x00000080 /* ioctl related information */
+#define D_BLOCKS      0x00000100 /* ext2 block allocation */
+#define D_NET         0x00000200 /* network communications */
+#define D_WARNING     0x00000400 /* CWARN(...) == CDEBUG (D_WARNING, ...) */
+#define D_BUFFS       0x00000800
+#define D_OTHER       0x00001000
+#define D_DENTRY      0x00002000
+#define D_PORTALS     0x00004000 /* ENTRY/EXIT markers */
+#define D_PAGE        0x00008000 /* bulk page handling */
+#define D_DLMTRACE    0x00010000
+#define D_ERROR       0x00020000 /* CERROR(...) == CDEBUG (D_ERROR, ...) */
+#define D_EMERG       0x00040000 /* CEMERG(...) == CDEBUG (D_EMERG, ...) */
+#define D_HA          0x00080000 /* recovery and failover */
+#define D_RPCTRACE    0x00100000 /* for distributed debugging */
+#define D_VFSTRACE    0x00200000
+#define D_READA       0x00400000 /* read-ahead */
 
 #ifdef __KERNEL__
 # include <linux/sched.h> /* THREAD_SIZE */
 #else
-# define THREAD_SIZE 8192
+# ifndef THREAD_SIZE /* x86_64 has THREAD_SIZE in userspace */
+#  define THREAD_SIZE 8192
+# endif
 #endif
 
+#define LUSTRE_TRACE_SIZE (THREAD_SIZE >> 5)
+
 #ifdef __KERNEL__
 # ifdef  __ia64__
 #  define CDEBUG_STACK (THREAD_SIZE -                                      \
@@ -90,10 +97,9 @@ extern unsigned int portal_printk;
 #define CHECK_STACK(stack)                                                    \
         do {                                                                  \
                 if ((stack) > 3*THREAD_SIZE/4 && (stack) > portal_stack) {    \
-                        portals_debug_msg(DEBUG_SUBSYSTEM, D_ERROR,           \
+                        portals_debug_msg(DEBUG_SUBSYSTEM, D_WARNING,         \
                                           __FILE__, __FUNCTION__, __LINE__,   \
-                                          (stack),                            \
-                                          "maximum lustre stack %u\n",        \
+                                          (stack),"maximum lustre stack %u\n",\
                                           portal_stack = (stack));            \
                       /*panic("LBUG");*/                                      \
                 }                                                             \
@@ -106,8 +112,10 @@ extern unsigned int portal_printk;
 #if 1
 #define CDEBUG(mask, format, a...)                                            \
 do {                                                                          \
+        if (portal_cerror == 0)                                               \
+                break;                                                        \
         CHECK_STACK(CDEBUG_STACK);                                            \
-        if (!(mask) || ((mask) & (D_ERROR | D_EMERG)) ||                      \
+        if (((mask) & (D_ERROR | D_EMERG | D_WARNING)) ||                     \
             (portal_debug & (mask) &&                                         \
              portal_subsystem_debug & DEBUG_SUBSYSTEM))                       \
                 portals_debug_msg(DEBUG_SUBSYSTEM, mask,                      \
@@ -176,7 +184,6 @@ do {                                                                          \
         PREPARE_TQUEUE((wq), (cb), (cbdata));                                 \
 } while (0)
 
-#define ll_invalidate_inode_pages invalidate_inode_pages
 #define PageUptodate Page_Uptodate
 #define our_recalc_sigpending(current) recalc_sigpending(current)
 #define num_online_cpus() smp_num_cpus
@@ -185,6 +192,7 @@ static inline void our_cond_resched(void)
         if (current->need_resched)
                schedule ();
 }
+#define work_struct_t       struct tq_struct
 
 #else
 
@@ -192,7 +200,6 @@ static inline void our_cond_resched(void)
 do {                                                                          \
         INIT_WORK((wq), (void *)(cb), (void *)(cbdata));                      \
 } while (0)
-#define ll_invalidate_inode_pages(inode) invalidate_inode_pages((inode)->i_mapping)
 #define wait_on_page wait_on_page_locked
 #define our_recalc_sigpending(current) recalc_sigpending()
 #define strtok(a,b) strpbrk(a, b)
@@ -200,6 +207,8 @@ static inline void our_cond_resched(void)
 {
         cond_resched();
 }
+#define work_struct_t      struct work_struct
+
 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0) */
 
 #ifdef PORTAL_DEBUG
@@ -207,8 +216,28 @@ extern void kportal_assertion_failed(char *expr, char *file, const char *func,
                                      const int line);
 #define LASSERT(e) ((e) ? 0 : kportal_assertion_failed( #e , __FILE__,  \
                                                         __FUNCTION__, __LINE__))
+/* it would be great to dump_stack() here, but some kernels
+ * export it as show_stack() and I can't be bothered to
+ * proprely engage in that dance right now */
+#define LASSERTF(cond, fmt...)                                                \
+        do {                                                                  \
+                if (unlikely(!(cond))) {                                      \
+                        portals_debug_msg(0, D_EMERG,  __FILE__, __FUNCTION__,\
+                                          __LINE__,  CDEBUG_STACK,            \
+                                          "ASSERTION(" #cond ") failed:" fmt);\
+                        LBUG();                                               \
+                }                                                             \
+        } while (0)
+
 #else
 #define LASSERT(e)
+#define LASSERTF(cond, fmt...) do { } while (0)
+#endif
+
+#ifdef CONFIG_SMP
+#define LASSERT_SPIN_LOCKED(lock) LASSERT(spin_is_locked(lock))
+#else
+#define LASSERT_SPIN_LOCKED(lock) do {} while(0)
 #endif
 
 #ifdef __arch_um__
@@ -254,17 +283,20 @@ do {                                                                          \
 
 #define PORTAL_VMALLOC_SIZE        16384
 
-#define PORTAL_ALLOC(ptr, size)                                           \
+#define PORTAL_ALLOC_GFP(ptr, size, mask)                                 \
 do {                                                                      \
-        LASSERT (!in_interrupt());                                        \
+        LASSERT(!in_interrupt() ||                                        \
+               (size <= PORTAL_VMALLOC_SIZE && mask == GFP_ATOMIC));      \
         if ((size) > PORTAL_VMALLOC_SIZE)                                 \
                 (ptr) = vmalloc(size);                                    \
         else                                                              \
-                (ptr) = kmalloc((size), GFP_NOFS);                        \
-        if ((ptr) == NULL)                                                \
+                (ptr) = kmalloc((size), (mask));                          \
+        if ((ptr) == NULL) {                                              \
                 CERROR("PORTALS: out of memory at %s:%d (tried to alloc '"\
                        #ptr "' = %d)\n", __FILE__, __LINE__, (int)(size));\
-        else {                                                            \
+                CERROR("PORTALS: %d total bytes allocated by portals\n",  \
+                       atomic_read(&portal_kmemory));                     \
+        } else {                                                          \
                 portal_kmem_inc((ptr), (size));                           \
                 memset((ptr), 0, (size));                                 \
         }                                                                 \
@@ -272,6 +304,12 @@ do {                                                                      \
                (int)(size), (ptr), atomic_read (&portal_kmemory));        \
 } while (0)
 
+#define PORTAL_ALLOC(ptr, size) \
+        PORTAL_ALLOC_GFP(ptr, size, GFP_NOFS)
+
+#define PORTAL_ALLOC_ATOMIC(ptr, size) \
+        PORTAL_ALLOC_GFP(ptr, size, GFP_ATOMIC)
+
 #define PORTAL_FREE(ptr, size)                                          \
 do {                                                                    \
         int s = (size);                                                 \
@@ -289,37 +327,6 @@ do {                                                                    \
                s, (ptr), atomic_read(&portal_kmemory));                 \
 } while (0)
 
-#define PORTAL_SLAB_ALLOC(ptr, slab, size)                                \
-do {                                                                      \
-        LASSERT(!in_interrupt());                                         \
-        (ptr) = kmem_cache_alloc((slab), SLAB_KERNEL);                    \
-        if ((ptr) == NULL) {                                              \
-                CERROR("PORTALS: out of memory at %s:%d (tried to alloc"  \
-                       " '" #ptr "' from slab '" #slab "')\n", __FILE__,  \
-                       __LINE__);                                         \
-        } else {                                                          \
-                portal_kmem_inc((ptr), (size));                           \
-                memset((ptr), 0, (size));                                 \
-        }                                                                 \
-        CDEBUG(D_MALLOC, "kmalloced '" #ptr "': %ld at %p (tot %d).\n",   \
-               (int)(size), (ptr), atomic_read(&portal_kmemory));         \
-} while (0)
-
-#define PORTAL_SLAB_FREE(ptr, slab, size)                               \
-do {                                                                    \
-        int s = (size);                                                 \
-        if ((ptr) == NULL) {                                            \
-                CERROR("PORTALS: free NULL '" #ptr "' (%d bytes) at "   \
-                       "%s:%d\n", s, __FILE__, __LINE__);               \
-                break;                                                  \
-        }                                                               \
-        memset((ptr), 0x5a, s);                                         \
-        kmem_cache_free((slab), ptr);                                   \
-        portal_kmem_dec((ptr), s);                                      \
-        CDEBUG(D_MALLOC, "kfreed '" #ptr "': %d at %p (tot %d).\n",     \
-               s, (ptr), atomic_read (&portal_kmemory));                \
-} while (0)
-
 /* ------------------------------------------------------------------- */
 
 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2,5,0))
@@ -361,22 +368,25 @@ typedef struct {
         struct list_head     kprfd_list;        /* stash in queues (routing target can use) */
         ptl_nid_t            kprfd_target_nid;  /* final destination NID */
         ptl_nid_t            kprfd_gateway_nid; /* gateway NID */
-        int                  kprfd_nob;         /* # message bytes (including header) */
-        int                  kprfd_niov;        /* # message frags (including header) */
-        struct iovec        *kprfd_iov;         /* message fragments */
-        void                *kprfd_router_arg;  // originating NAL's router arg
+        ptl_hdr_t           *kprfd_hdr;         /* header in wire byte order */
+        int                  kprfd_nob;         /* # payload bytes */
+        int                  kprfd_niov;        /* # payload frags */
+        ptl_kiov_t          *kprfd_kiov;        /* payload fragments */
+        void                *kprfd_router_arg;  /* originating NAL's router arg */
         kpr_fwd_callback_t   kprfd_callback;    /* completion callback */
         void                *kprfd_callback_arg; /* completion callback arg */
-        kprfd_scratch_t      kprfd_scratch;    // scratchpad for routing targets
+        kprfd_scratch_t      kprfd_scratch;     /* scratchpad for routing targets */
 } kpr_fwd_desc_t;
 
 typedef void  (*kpr_fwd_t)(void *arg, kpr_fwd_desc_t *fwd);
+typedef void  (*kpr_notify_t)(void *arg, ptl_nid_t peer, int alive);
 
 /* NAL's routing interface (Kernel Portals Routing Nal Interface) */
 typedef const struct {
         int             kprni_nalid;    /* NAL's id */
         void           *kprni_arg;      /* Arg to pass when calling into NAL */
         kpr_fwd_t       kprni_fwd;      /* NAL's forwarding entrypoint */
+        kpr_notify_t    kprni_notify;   /* NAL's notification entrypoint */
 } kpr_nal_interface_t;
 
 /* Router's routing interface (Kernel Portals Routing Router Interface) */
@@ -386,9 +396,10 @@ typedef const struct {
         int     (*kprri_register) (kpr_nal_interface_t *nal_interface,
                                    void **router_arg);
 
-        /* ask the router to find a gateway that forwards to 'nid' and is a peer
-         * of the calling NAL */
-        int     (*kprri_lookup) (void *router_arg, ptl_nid_t nid,
+        /* ask the router to find a gateway that forwards to 'nid' and is a
+         * peer of the calling NAL; assume caller will send 'nob' bytes of
+         * payload there */
+        int     (*kprri_lookup) (void *router_arg, ptl_nid_t nid, int nob,
                                  ptl_nid_t *gateway_nid);
 
         /* hand a packet over to the router for forwarding */
@@ -398,6 +409,10 @@ typedef const struct {
         void    (*kprri_fwd_done) (void *router_arg, kpr_fwd_desc_t *fwd,
                                    int error);
 
+        /* notify the router about peer state */
+        void    (*kprri_notify) (void *router_arg, ptl_nid_t peer,
+                                 int alive, time_t when);
+
         /* the calling NAL is shutting down */
         void    (*kprri_shutdown) (void *router_arg);
 
@@ -416,10 +431,14 @@ typedef struct {
 typedef const struct {
         int     (*kprci_add_route)(int gateway_nal, ptl_nid_t gateway_nid,
                                    ptl_nid_t lo_nid, ptl_nid_t hi_nid);
-        int     (*kprci_del_route)(ptl_nid_t nid);
+        int     (*kprci_del_route)(int gateway_nal, ptl_nid_t gateway_nid,
+                                   ptl_nid_t lo_nid, ptl_nid_t hi_nid);
         int     (*kprci_get_route)(int index, int *gateway_nal,
-                                   ptl_nid_t *gateway, ptl_nid_t *lo_nid,
-                                   ptl_nid_t *hi_nid);
+                                   ptl_nid_t *gateway,
+                                   ptl_nid_t *lo_nid, ptl_nid_t *hi_nid,
+                                   int *alive);
+        int     (*kprci_notify)(int gateway_nal, ptl_nid_t gateway_nid,
+                                int alive, time_t when);
 } kpr_control_interface_t;
 
 extern kpr_control_interface_t  kpr_control_interface;
@@ -449,25 +468,26 @@ kpr_routing (kpr_router_t *router)
 }
 
 static inline int
-kpr_lookup (kpr_router_t *router, ptl_nid_t nid, ptl_nid_t *gateway_nid)
+kpr_lookup (kpr_router_t *router, ptl_nid_t nid, int nob, ptl_nid_t *gateway_nid)
 {
         if (!kpr_routing (router))
-                return (-EHOSTUNREACH);
+                return (-ENETUNREACH);
 
-        return (router->kpr_interface->kprri_lookup(router->kpr_arg, nid,
+        return (router->kpr_interface->kprri_lookup(router->kpr_arg, nid, nob,
                                                     gateway_nid));
 }
 
 static inline void
-kpr_fwd_init (kpr_fwd_desc_t *fwd, ptl_nid_t nid,
-              int nob, int niov, struct iovec *iov,
+kpr_fwd_init (kpr_fwd_desc_t *fwd, ptl_nid_t nid, ptl_hdr_t *hdr,
+              int nob, int niov, ptl_kiov_t *kiov,
               kpr_fwd_callback_t callback, void *callback_arg)
 {
         fwd->kprfd_target_nid   = nid;
         fwd->kprfd_gateway_nid  = nid;
+        fwd->kprfd_hdr          = hdr;
         fwd->kprfd_nob          = nob;
         fwd->kprfd_niov         = niov;
-        fwd->kprfd_iov          = iov;
+        fwd->kprfd_kiov         = kiov;
         fwd->kprfd_callback     = callback;
         fwd->kprfd_callback_arg = callback_arg;
 }
@@ -476,7 +496,7 @@ static inline void
 kpr_fwd_start (kpr_router_t *router, kpr_fwd_desc_t *fwd)
 {
         if (!kpr_routing (router))
-                fwd->kprfd_callback (fwd->kprfd_callback_arg, -EHOSTUNREACH);
+                fwd->kprfd_callback (fwd->kprfd_callback_arg, -ENETUNREACH);
         else
                 router->kpr_interface->kprri_fwd_start (router->kpr_arg, fwd);
 }
@@ -489,6 +509,16 @@ kpr_fwd_done (kpr_router_t *router, kpr_fwd_desc_t *fwd, int error)
 }
 
 static inline void
+kpr_notify (kpr_router_t *router,
+            ptl_nid_t peer, int alive, time_t when)
+{
+        if (!kpr_routing (router))
+                return;
+
+        router->kpr_interface->kprri_notify(router->kpr_arg, peer, alive, when);
+}
+
+static inline void
 kpr_shutdown (kpr_router_t *router)
 {
         if (kpr_routing (router))
@@ -554,6 +584,10 @@ extern struct prof_ent prof_ents[MAX_PROFS];
 #endif /* PORTALS_PROFILING */
 
 /* debug.c */
+extern spinlock_t stack_backtrace_lock;
+
+char *portals_debug_dumpstack(void);
+void portals_run_upcall(char **argv);
 void portals_run_lbug_upcall(char * file, const char *fn, const int line);
 void portals_debug_dumplog(void);
 int portals_debug_init(unsigned long bufsize);
@@ -574,7 +608,7 @@ __s32 portals_debug_copy_to_user(char *buf, unsigned long len);
 #endif
 void portals_debug_msg(int subsys, int mask, char *file, const char *fn,
                        const int line, unsigned long stack,
-                       const char *format, ...)
+                       char *format, ...)
         __attribute__ ((format (printf, 7, 8)));
 #else
 void portals_debug_msg(int subsys, int mask, char *file, const char *fn,
@@ -595,6 +629,8 @@ extern void kportal_blockallsigs (void);
 # include <stdlib.h>
 #ifndef __CYGWIN__
 # include <stdint.h>
+#else
+# include <cygwin-ioctl.h>
 #endif
 # include <unistd.h>
 # include <time.h>
@@ -606,8 +642,10 @@ extern void kportal_blockallsigs (void);
 #  undef NDEBUG
 #  include <assert.h>
 #  define LASSERT(e)     assert(e)
+#  define LASSERTF(cond, args...)     assert(cond)
 # else
 #  define LASSERT(e)
+#  define LASSERTF(cond, args...) do { } while (0)
 # endif
 # define printk(format, args...) printf (format, ## args)
 # define PORTAL_ALLOC(ptr, size) do { (ptr) = malloc(size); } while (0);
@@ -618,16 +656,150 @@ extern void kportal_blockallsigs (void);
            getpid() , stack, ## a);
 #endif
 
+/* support decl needed both by kernel and liblustre */
+char *portals_nid2str(int nal, ptl_nid_t nid, char *str);
+
 #ifndef CURRENT_TIME
 # define CURRENT_TIME time(0)
 #endif
 
+/******************************************************************************/
+/* Light-weight trace
+ * Support for temporary event tracing with minimal Heisenberg effect. */
+#define LWT_SUPPORT  0
+
+#define LWT_MEMORY   (64<<20)
+#define LWT_MAX_CPUS 4
+
+typedef struct {
+        cycles_t    lwte_when;
+        char       *lwte_where;
+        void       *lwte_task;
+        long        lwte_p1;
+        long        lwte_p2;
+        long        lwte_p3;
+        long        lwte_p4;
+#if BITS_PER_LONG > 32
+        long        lwte_pad;
+#endif
+} lwt_event_t;
+
+#if LWT_SUPPORT
+#ifdef __KERNEL__
+#define LWT_EVENTS_PER_PAGE (PAGE_SIZE / sizeof (lwt_event_t))
+
+typedef struct _lwt_page {
+        struct list_head     lwtp_list;
+        struct page         *lwtp_page;
+        lwt_event_t         *lwtp_events;
+} lwt_page_t;
+
+typedef struct {
+        int                lwtc_current_index;
+        lwt_page_t        *lwtc_current_page;
+} lwt_cpu_t;
+
+extern int       lwt_enabled;
+extern lwt_cpu_t lwt_cpus[];
+
+extern int  lwt_init (void);
+extern void lwt_fini (void);
+extern int  lwt_lookup_string (int *size, char *knlptr,
+                               char *usrptr, int usrsize);
+extern int  lwt_control (int enable, int clear);
+extern int  lwt_snapshot (cycles_t *now, int *ncpu, int *total_size,
+                          void *user_ptr, int user_size);
+
+/* Note that we _don't_ define LWT_EVENT at all if LWT_SUPPORT isn't set.
+ * This stuff is meant for finding specific problems; it never stays in
+ * production code... */
+
+#define LWTSTR(n)       #n
+#define LWTWHERE(f,l)   f ":" LWTSTR(l)
+
+#define LWT_EVENT(p1, p2, p3, p4)                                       \
+do {                                                                    \
+        unsigned long    flags;                                         \
+        lwt_cpu_t       *cpu;                                           \
+        lwt_page_t      *p;                                             \
+        lwt_event_t     *e;                                             \
+                                                                        \
+        local_irq_save (flags);                                         \
+                                                                        \
+        if (lwt_enabled) {                                              \
+                cpu = &lwt_cpus[smp_processor_id()];                    \
+                p = cpu->lwtc_current_page;                             \
+                e = &p->lwtp_events[cpu->lwtc_current_index++];         \
+                                                                        \
+                if (cpu->lwtc_current_index >= LWT_EVENTS_PER_PAGE) {   \
+                        cpu->lwtc_current_page =                        \
+                                list_entry (p->lwtp_list.next,          \
+                                            lwt_page_t, lwtp_list);     \
+                        cpu->lwtc_current_index = 0;                    \
+                }                                                       \
+                                                                        \
+                e->lwte_when  = get_cycles();                           \
+                e->lwte_where = LWTWHERE(__FILE__,__LINE__);            \
+                e->lwte_task  = current;                                \
+                e->lwte_p1    = (long)(p1);                             \
+                e->lwte_p2    = (long)(p2);                             \
+                e->lwte_p3    = (long)(p3);                             \
+                e->lwte_p4    = (long)(p4);                             \
+        }                                                               \
+                                                                        \
+        local_irq_restore (flags);                                      \
+} while (0)
+#else  /* __KERNEL__ */
+#define LWT_EVENT(p1,p2,p3,p4)     /* no userland implementation yet */
+#endif /* __KERNEL__ */
+#endif /* LWT_SUPPORT */
+
+struct portals_device_userstate
+{
+        int          pdu_memhog_pages;
+        struct page *pdu_memhog_root_page;
+};
+
 #include <linux/portals_lib.h>
 
 /*
  * USER LEVEL STUFF BELOW
  */
 
+#define PORTALS_CFG_VERSION 0x00010001;
+
+struct portals_cfg {
+        __u32 pcfg_version;
+        __u32 pcfg_command;
+
+        __u32 pcfg_nal;
+        __u32 pcfg_flags;
+
+        __u32 pcfg_gw_nal;
+        __u64 pcfg_nid;
+        __u64 pcfg_nid2;
+        __u64 pcfg_nid3;
+        __u32 pcfg_id;
+        __u32 pcfg_misc;
+        __u32 pcfg_fd;
+        __u32 pcfg_count;
+        __u32 pcfg_size;
+        __u32 pcfg_wait;
+
+        __u32 pcfg_plen1; /* buffers in userspace */
+        char *pcfg_pbuf1;
+        __u32 pcfg_plen2; /* buffers in userspace */
+        char *pcfg_pbuf2;
+};
+
+#define PCFG_INIT(pcfg, cmd)                            \
+do {                                                    \
+        memset(&pcfg, 0, sizeof(pcfg));                 \
+        pcfg.pcfg_version = PORTALS_CFG_VERSION;        \
+        pcfg.pcfg_command = (cmd);                      \
+                                                        \
+} while (0)
+
 #define PORTAL_IOCTL_VERSION 0x00010007
 #define PING_SYNC       0
 #define PING_ASYNC      1
@@ -849,34 +1021,38 @@ static inline int portal_ioctl_getdata(char *buf, char *end, void *arg)
 #define IOC_PORTAL_CLEAR_DEBUG             _IOWR('e', 32, long)
 #define IOC_PORTAL_MARK_DEBUG              _IOWR('e', 33, long)
 #define IOC_PORTAL_PANIC                   _IOWR('e', 34, long)
-#define IOC_PORTAL_ADD_ROUTE               _IOWR('e', 35, long)
-#define IOC_PORTAL_DEL_ROUTE               _IOWR('e', 36, long)
-#define IOC_PORTAL_GET_ROUTE               _IOWR('e', 37, long)
-#define IOC_PORTAL_NAL_CMD                _IOWR('e', 38, long)
-#define IOC_PORTAL_GET_NID                 _IOWR('e', 39, long)
-#define IOC_PORTAL_FAIL_NID                _IOWR('e', 40, long)
-#define IOC_PORTAL_SET_DAEMON              _IOWR('e', 41, long)
-
-#define IOC_PORTAL_MAX_NR               41
+#define IOC_PORTAL_NAL_CMD                 _IOWR('e', 35, long)
+#define IOC_PORTAL_GET_NID                 _IOWR('e', 36, long)
+#define IOC_PORTAL_FAIL_NID                _IOWR('e', 37, long)
+#define IOC_PORTAL_SET_DAEMON              _IOWR('e', 38, long)
+#define IOC_PORTAL_LWT_CONTROL             _IOWR('e', 39, long)
+#define IOC_PORTAL_LWT_SNAPSHOT            _IOWR('e', 40, long)
+#define IOC_PORTAL_LWT_LOOKUP_STRING       _IOWR('e', 41, long)
+#define IOC_PORTAL_MEMHOG                  _IOWR('e', 42, long)
+#define IOC_PORTAL_MAX_NR                             42
 
 enum {
-        QSWNAL  =  1,
-        SOCKNAL,
-        GMNAL,
-        TOENAL,
-        TCPNAL,
-        SCIMACNAL,
+        QSWNAL    = 1,
+        SOCKNAL   = 2,
+        GMNAL     = 3,
+        /*          4 unused */
+        TCPNAL    = 5,
+        SCIMACNAL = 6,
+        ROUTER    = 7,
+        IBNAL     = 8,
         NAL_ENUM_END_MARKER
 };
 
 #ifdef __KERNEL__
 extern ptl_handle_ni_t  kqswnal_ni;
 extern ptl_handle_ni_t  ksocknal_ni;
-extern ptl_handle_ni_t  ktoenal_ni;
 extern ptl_handle_ni_t  kgmnal_ni;
+extern ptl_handle_ni_t  kibnal_ni;
 extern ptl_handle_ni_t  kscimacnal_ni;
 #endif
 
+#define PTL_NALFMT_SIZE         16
+
 #define NAL_MAX_NR (NAL_ENUM_END_MARKER - 1)
 
 #define NAL_CMD_REGISTER_PEER_FD     100
@@ -887,6 +1063,11 @@ extern ptl_handle_ni_t  kscimacnal_ni;
 #define NAL_CMD_DEL_AUTOCONN         105
 #define NAL_CMD_ADD_AUTOCONN         106
 #define NAL_CMD_GET_AUTOCONN         107
+#define NAL_CMD_GET_TXDESC           108
+#define NAL_CMD_ADD_ROUTE            109
+#define NAL_CMD_DEL_ROUTE            110
+#define NAL_CMD_GET_ROUTE            111
+#define NAL_CMD_NOTIFY_ROUTER        112
 
 enum {
         DEBUG_DAEMON_START       =  1,
@@ -901,11 +1082,20 @@ struct lustre_peer {
         ptl_handle_ni_t peer_ni;
 };
 
+
 /* module.c */
-typedef int (*nal_cmd_handler_t)(struct portal_ioctl_data *, void * private);
+typedef int (*nal_cmd_handler_t)(struct portals_cfg *, void * private);
 int kportal_nal_register(int nal, nal_cmd_handler_t handler, void * private);
 int kportal_nal_unregister(int nal);
 
+enum cfg_record_type {
+        PORTALS_CFG_TYPE = 1,
+        LUSTRE_CFG_TYPE = 123,
+};
+
+typedef int (*cfg_record_cb_t)(enum cfg_record_type, int len, void *data);
+int kportal_nal_cmd(struct portals_cfg *);
+
 ptl_handle_ni_t *kportal_get_ni (int nal);
 void kportal_put_ni (int nal);
 
@@ -919,14 +1109,19 @@ void kportal_put_ni (int nal);
 # endif
 #endif
 
-#if (BITS_PER_LONG == 32 || __WORDSIZE == 32)
+#if defined(__x86_64__)
+# define LPU64 "%Lu"
+# define LPD64 "%Ld"
+# define LPX64 "%#Lx"
+# define LPSZ  "%lu"
+# define LPSSZ "%ld"
+#elif (BITS_PER_LONG == 32 || __WORDSIZE == 32)
 # define LPU64 "%Lu"
 # define LPD64 "%Ld"
 # define LPX64 "%#Lx"
 # define LPSZ  "%u"
 # define LPSSZ "%d"
-#endif
-#if (BITS_PER_LONG == 64 || __WORDSIZE == 64)
+#elif (BITS_PER_LONG == 64 || __WORDSIZE == 64)
 # define LPU64 "%lu"
 # define LPD64 "%ld"
 # define LPX64 "%#lx"