From 04ae51bbf09b278161924819be0a7c8727efbd41 Mon Sep 17 00:00:00 2001 From: eeb Date: Mon, 4 Apr 2005 10:09:43 +0000 Subject: [PATCH] * 5852 vibnal patch to detect __u64/void* vv_scatgat_t::v_address and use appropriately for HIGHMEM support --- lnet/autoconf/lustre-lnet.m4 | 30 ++++++++++++++++++++++++++++++ lnet/klnds/viblnd/viblnd.c | 2 +- lnet/klnds/viblnd/viblnd.h | 35 +++++++++++++++++++++++++++++++++++ lnet/klnds/viblnd/viblnd_cb.c | 20 ++++++++------------ 4 files changed, 74 insertions(+), 13 deletions(-) diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4 index 0768673..4611dee 100644 --- a/lnet/autoconf/lustre-lnet.m4 +++ b/lnet/autoconf/lustre-lnet.m4 @@ -367,6 +367,36 @@ else ]) EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save" fi +if test -n "$VIBNAL"; then + AC_MSG_CHECKING([if Voltaire still uses void * sg addresses]) + EXTRA_KCFLAGS_save="$EXTRA_KCFLAGS" + EXTRA_KCFLAGS="$EXTRA_KCFLAGS $VIBCPPFLAGS" + LB_LINUX_TRY_COMPILE([ + #include + #include + #ifdef __BIG_ENDIAN + # define CPU_BE 1 + # define CPU_LE 0 + #endif + #ifdef __LITTLE_ENDIAN + # define CPU_BE 0 + # define CPU_LE 1 + #endif + #include + #include + #include + ],[ + vv_scatgat_t sg; + + return &sg.v_address[3] == NULL; + ],[ + AC_MSG_RESULT([yes]) + VIBCPPFLAGS="$VIBCPPFLAGS -DIBNAL_VOIDSTAR_SGADDR=1" + ],[ + AC_MSG_RESULT([no]) + ]) + EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save" +fi AC_SUBST(VIBCPPFLAGS) AC_SUBST(VIBNAL) ]) diff --git a/lnet/klnds/viblnd/viblnd.c b/lnet/klnds/viblnd/viblnd.c index 41e0efc..7995610e 100644 --- a/lnet/klnds/viblnd/viblnd.c +++ b/lnet/klnds/viblnd/viblnd.c @@ -1297,7 +1297,7 @@ kibnal_alloc_pages (kib_pages_t **pp, int npages, int allow_write) for (i = 0; i < npages; i++) { phys_pages[i].size = PAGE_SIZE; - phys_pages[i].start = page_to_phys(p->ibp_pages[i]); + phys_pages[i].start = kibnal_page2phys(p->ibp_pages[i]); } VV_ACCESS_CONTROL_MASK_SET_ALL(access); diff --git a/lnet/klnds/viblnd/viblnd.h b/lnet/klnds/viblnd/viblnd.h index 46d3be6..ff28bd6 100644 --- a/lnet/klnds/viblnd/viblnd.h +++ b/lnet/klnds/viblnd/viblnd.h @@ -528,6 +528,41 @@ kibnal_queue_tx_locked (kib_tx_t *tx, kib_conn_t *conn) list_add_tail(&tx->tx_list, &conn->ibc_tx_queue); } +static inline __u64 +kibnal_page2phys (struct page *p) +{ + return ((__u64)(p - mem_map)) << PAGE_SHIFT; +} + +#if IBNAL_VOIDSTAR_SGADDR +# if CONFIG_HIGHMEM +# error "Can't support HIGHMEM when vv_scatgat_t::v_address is void *" +# endif +# define KIBNAL_ADDR2SG(a) ((void *)((unsigned long)(a))) +# define KIBNAL_SG2ADDR(a) ((__u64)((unsigned long)(a))) +static inline __u64 kibnal_addr2net (__u64 addr) +{ + void *netaddr; + vv_return_t vvrc = vv_va2advertise_addr(kibnal_data.kib_hca, + KIBNAL_ADDR2SG(addr), + &netaddr); + LASSERT (vvrc == vv_return_ok); + return KIBNAL_SG2ADDR(netaddr); +} +#else +# define KIBNAL_ADDR2SG(a) a +# define KIBNAL_SG2ADDR(a) a +static inline __u64 kibnal_addr2net (__u64 addr) +{ + __u64 netaddr; + vv_return_t vvrc = vv_va2advertise_addr(kibnal_data.kib_hca, + addr, + &netaddr); + LASSERT (vvrc == vv_return_ok); + return netaddr; +} +#endif + /* CAVEAT EMPTOR: We rely on tx/rx descriptor alignment to allow us to use the * lowest 2 bits of the work request id to stash the work item type (the op * field is not valid when the wc completes in error). */ diff --git a/lnet/klnds/viblnd/viblnd_cb.c b/lnet/klnds/viblnd/viblnd_cb.c index ba021bf..c6d8bf8 100644 --- a/lnet/klnds/viblnd/viblnd_cb.c +++ b/lnet/klnds/viblnd/viblnd_cb.c @@ -154,7 +154,7 @@ kibnal_post_rx (kib_rx_t *rx, int credit) LASSERT (!in_interrupt()); rx->rx_gl = (vv_scatgat_t) { - .v_address = (void *)((unsigned long)KIBNAL_RX_VADDR(rx)), + .v_address = KIBNAL_ADDR2SG(KIBNAL_RX_VADDR(rx)), .l_key = KIBNAL_RX_LKEY(rx), .length = IBNAL_MSG_SIZE, }; @@ -170,10 +170,10 @@ kibnal_post_rx (kib_rx_t *rx, int credit) LASSERT (conn->ibc_state >= IBNAL_CONN_INIT); LASSERT (!rx->rx_posted); - CDEBUG(D_NET, "posting rx [%d %x %p]\n", + CDEBUG(D_NET, "posting rx [%d %x "LPX64"]\n", rx->rx_wrq.scatgat_list->length, rx->rx_wrq.scatgat_list->l_key, - rx->rx_wrq.scatgat_list->v_address); + KIBNAL_SG2ADDR(rx->rx_wrq.scatgat_list->v_address)); if (conn->ibc_state > IBNAL_CONN_ESTABLISHED) { /* No more posts for this rx; so lose its ref */ @@ -525,12 +525,9 @@ kibnal_append_rdfrag(kib_rdma_desc_t *rd, int active, struct page *page, return -EMSGSIZE; } -#if CONFIG_HIGHMEM -# error "This probably doesn't work because of over/underflow when casting between __u64 and void *..." -#endif /* Try to create an address that adapter-tavor will munge into a valid * network address, given how it maps all phys mem into 1 region */ - addr = page_to_phys(page) + page_offset + PAGE_OFFSET; + addr = kibnal_page2phys(page) + page_offset + PAGE_OFFSET; vvrc = vv_get_gen_mr_attrib(kibnal_data.kib_hca, (void *)((unsigned long)addr), @@ -552,9 +549,8 @@ kibnal_append_rdfrag(kib_rdma_desc_t *rd, int active, struct page *page, CERROR ("> 1 key for single RDMA desc\n"); return -EINVAL; } - vv_va2advertise_addr(kibnal_data.kib_hca, - (void *)((unsigned long)addr), &ptr); - frag_addr = (unsigned long)ptr; + + frag_addr = kibnal_addr2net(addr); } kibnal_rf_set(frag, frag_addr, len); @@ -1091,7 +1087,7 @@ kibnal_init_tx_msg (kib_tx_t *tx, int type, int body_nob) kibnal_init_msg(tx->tx_msg, type, body_nob); *gl = (vv_scatgat_t) { - .v_address = (void *)((unsigned long)KIBNAL_TX_VADDR(tx)), + .v_address = KIBNAL_ADDR2SG(KIBNAL_TX_VADDR(tx)), .l_key = KIBNAL_TX_LKEY(tx), .length = nob, }; @@ -1162,7 +1158,7 @@ kibnal_init_rdma (kib_tx_t *tx, int type, int nob, wrknob = MIN(MIN(srcfrag->rf_nob, dstfrag->rf_nob), resid); gl = &tx->tx_gl[tx->tx_nwrq]; - gl->v_address = (void *)((unsigned long)kibnal_rf_addr(srcfrag)); + gl->v_address = KIBNAL_ADDR2SG(kibnal_rf_addr(srcfrag)); gl->length = wrknob; gl->l_key = srcrd->rd_key; -- 1.8.3.1