* 5852 vibnal patch to detect __u64/void* vv_scatgat_t::v_address and

    use appropriately for HIGHMEM support

diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4
index 0768673..4611dee 100644 (file)
--- a/lnet/autoconf/lustre-lnet.m4
+++ b/lnet/autoconf/lustre-lnet.m4
@@ -367,6 +367,36 @@ else
        ])
        EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save"
 fi
+if test -n "$VIBNAL"; then
+       AC_MSG_CHECKING([if Voltaire still uses void * sg addresses])
+       EXTRA_KCFLAGS_save="$EXTRA_KCFLAGS"
+       EXTRA_KCFLAGS="$EXTRA_KCFLAGS $VIBCPPFLAGS"
+       LB_LINUX_TRY_COMPILE([
+               #include <linux/list.h>
+               #include <asm/byteorder.h>
+               #ifdef __BIG_ENDIAN
+               # define CPU_BE 1
+                # define CPU_LE 0
+               #endif
+               #ifdef __LITTLE_ENDIAN
+               # define CPU_BE 0
+               # define CPU_LE 1
+               #endif
+               #include <vverbs.h>
+               #include <ib-cm.h>
+               #include <ibat.h>
+       ],[
+               vv_scatgat_t  sg;
+
+               return &sg.v_address[3] == NULL;
+       ],[
+               AC_MSG_RESULT([yes])
+               VIBCPPFLAGS="$VIBCPPFLAGS -DIBNAL_VOIDSTAR_SGADDR=1"
+       ],[
+               AC_MSG_RESULT([no])
+       ])
+       EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save"
+fi
 AC_SUBST(VIBCPPFLAGS)
 AC_SUBST(VIBNAL)
 ])

diff --git a/lnet/klnds/viblnd/viblnd.c b/lnet/klnds/viblnd/viblnd.c
index 41e0efc..7995610 100644 (file)
--- a/lnet/klnds/viblnd/viblnd.c
+++ b/lnet/klnds/viblnd/viblnd.c
@@ -1297,7 +1297,7 @@ kibnal_alloc_pages (kib_pages_t **pp, int npages, int allow_write)
 
         for (i = 0; i < npages; i++) {
                 phys_pages[i].size = PAGE_SIZE;
-                phys_pages[i].start = page_to_phys(p->ibp_pages[i]);
+                phys_pages[i].start = kibnal_page2phys(p->ibp_pages[i]);
         }
 
         VV_ACCESS_CONTROL_MASK_SET_ALL(access);

diff --git a/lnet/klnds/viblnd/viblnd.h b/lnet/klnds/viblnd/viblnd.h
index 46d3be6..ff28bd6 100644 (file)
--- a/lnet/klnds/viblnd/viblnd.h
+++ b/lnet/klnds/viblnd/viblnd.h
@@ -528,6 +528,41 @@ kibnal_queue_tx_locked (kib_tx_t *tx, kib_conn_t *conn)
         list_add_tail(&tx->tx_list, &conn->ibc_tx_queue);
 }
 
+static inline __u64
+kibnal_page2phys (struct page *p)
+{
+        return ((__u64)(p - mem_map)) << PAGE_SHIFT;
+}
+
+#if IBNAL_VOIDSTAR_SGADDR
+# if CONFIG_HIGHMEM
+#  error "Can't support HIGHMEM when vv_scatgat_t::v_address is void *"
+# endif
+# define KIBNAL_ADDR2SG(a)       ((void *)((unsigned long)(a)))
+# define KIBNAL_SG2ADDR(a)       ((__u64)((unsigned long)(a)))
+static inline __u64 kibnal_addr2net (__u64 addr)
+{
+        void        *netaddr;
+        vv_return_t  vvrc = vv_va2advertise_addr(kibnal_data.kib_hca, 
+                                                 KIBNAL_ADDR2SG(addr),
+                                                 &netaddr);
+        LASSERT (vvrc == vv_return_ok);
+        return KIBNAL_SG2ADDR(netaddr);
+}
+#else
+# define KIBNAL_ADDR2SG(a)       a
+# define KIBNAL_SG2ADDR(a)       a
+static inline __u64 kibnal_addr2net (__u64 addr)
+{
+        __u64        netaddr;
+        vv_return_t  vvrc = vv_va2advertise_addr(kibnal_data.kib_hca, 
+                                                 addr,
+                                                 &netaddr);
+        LASSERT (vvrc == vv_return_ok);
+        return netaddr;
+}
+#endif
+
 /* CAVEAT EMPTOR: We rely on tx/rx descriptor alignment to allow us to use the
  * lowest 2 bits of the work request id to stash the work item type (the op
  * field is not valid when the wc completes in error). */

diff --git a/lnet/klnds/viblnd/viblnd_cb.c b/lnet/klnds/viblnd/viblnd_cb.c
index ba021bf..c6d8bf8 100644 (file)
--- a/lnet/klnds/viblnd/viblnd_cb.c
+++ b/lnet/klnds/viblnd/viblnd_cb.c
@@ -154,7 +154,7 @@ kibnal_post_rx (kib_rx_t *rx, int credit)
         LASSERT (!in_interrupt());
         
         rx->rx_gl = (vv_scatgat_t) {
-                .v_address = (void *)((unsigned long)KIBNAL_RX_VADDR(rx)),
+                .v_address = KIBNAL_ADDR2SG(KIBNAL_RX_VADDR(rx)),
                 .l_key     = KIBNAL_RX_LKEY(rx),
                 .length    = IBNAL_MSG_SIZE,
         };
@@ -170,10 +170,10 @@ kibnal_post_rx (kib_rx_t *rx, int credit)
         LASSERT (conn->ibc_state >= IBNAL_CONN_INIT);
         LASSERT (!rx->rx_posted);
 
-        CDEBUG(D_NET, "posting rx [%d %x %p]\n", 
+        CDEBUG(D_NET, "posting rx [%d %x "LPX64"]\n", 
                rx->rx_wrq.scatgat_list->length,
                rx->rx_wrq.scatgat_list->l_key,
-               rx->rx_wrq.scatgat_list->v_address);
+               KIBNAL_SG2ADDR(rx->rx_wrq.scatgat_list->v_address));
 
         if (conn->ibc_state > IBNAL_CONN_ESTABLISHED) {
                 /* No more posts for this rx; so lose its ref */
@@ -525,12 +525,9 @@ kibnal_append_rdfrag(kib_rdma_desc_t *rd, int active, struct page *page,
                 return -EMSGSIZE;
         }
 
-#if CONFIG_HIGHMEM
-# error "This probably doesn't work because of over/underflow when casting between __u64 and void *..."
-#endif
         /* Try to create an address that adapter-tavor will munge into a valid
          * network address, given how it maps all phys mem into 1 region */
-        addr = page_to_phys(page) + page_offset + PAGE_OFFSET;
+        addr = kibnal_page2phys(page) + page_offset + PAGE_OFFSET;
 
         vvrc = vv_get_gen_mr_attrib(kibnal_data.kib_hca, 
                                     (void *)((unsigned long)addr),
@@ -552,9 +549,8 @@ kibnal_append_rdfrag(kib_rdma_desc_t *rd, int active, struct page *page,
                         CERROR ("> 1 key for single RDMA desc\n");
                         return -EINVAL;
                 }
-                vv_va2advertise_addr(kibnal_data.kib_hca, 
-                                     (void *)((unsigned long)addr), &ptr);
-                frag_addr = (unsigned long)ptr;
+
+                frag_addr = kibnal_addr2net(addr);
         }
 
         kibnal_rf_set(frag, frag_addr, len);
@@ -1091,7 +1087,7 @@ kibnal_init_tx_msg (kib_tx_t *tx, int type, int body_nob)
         kibnal_init_msg(tx->tx_msg, type, body_nob);
 
         *gl = (vv_scatgat_t) {
-                .v_address = (void *)((unsigned long)KIBNAL_TX_VADDR(tx)),
+                .v_address = KIBNAL_ADDR2SG(KIBNAL_TX_VADDR(tx)),
                 .l_key     = KIBNAL_TX_LKEY(tx),
                 .length    = nob,
         };
@@ -1162,7 +1158,7 @@ kibnal_init_rdma (kib_tx_t *tx, int type, int nob,
                 wrknob = MIN(MIN(srcfrag->rf_nob, dstfrag->rf_nob), resid);
 
                 gl = &tx->tx_gl[tx->tx_nwrq];
-                gl->v_address = (void *)((unsigned long)kibnal_rf_addr(srcfrag));
+                gl->v_address = KIBNAL_ADDR2SG(kibnal_rf_addr(srcfrag));
                 gl->length    = wrknob;
                 gl->l_key     = srcrd->rd_key;