From: liangzhen Date: Wed, 20 Jun 2007 17:43:52 +0000 (+0000) Subject: Add OFED1.2 support to o2iblnd X-Git-Tag: v1_7_100~55 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=8c9b78c42436facd5c163f8968394d9235094a70 Add OFED1.2 support to o2iblnd --- diff --git a/lnet/ChangeLog b/lnet/ChangeLog index c388239..959a299 100644 --- a/lnet/ChangeLog +++ b/lnet/ChangeLog @@ -4,7 +4,7 @@ socklnd - kernels up to 2.6.16 qswlnd - Qsnet kernel modules 5.20 and later openiblnd - IbGold 1.8.2 - o2iblnd - OFED 1.1 + o2iblnd - OFED 1.1 and 1.2 viblnd - Voltaire ibhost 3.4.5 and later ciblnd - Topspin 3.2.0 iiblnd - Infiniserv 3.3 + PathBits patch @@ -17,6 +17,21 @@ Severity : normal Bugzilla : 11680 Description: make panic on lbug configurable +Severity : major +Bugzilla : 12316 +Description: Add OFED1.2 support to o2iblnd +Details : o2iblnd depends on OFED's modules, if out-tree OFED's modules + are installed (other than kernel's in-tree infiniband), there + could be some problem while insmod o2iblnd (mismatch CRC of + ib_* symbols). + If extra Module.symvers is supported in kernel (i.e, 2.6.17), + this link provides solution: + https://bugs.openfabrics.org/show_bug.cgi?id=355 + if extra Module.symvers is not supported in kernel, we will + have to run the script in bug 12316 to update + $LINUX/module.symvers before building o2iblnd. + More details about this are in bug 12316. + ------------------------------------------------------------------------------ 2007-04-01 Cluster File Systems, Inc. diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4 index c5f1740..bec57e5 100644 --- a/lnet/autoconf/lustre-lnet.m4 +++ b/lnet/autoconf/lustre-lnet.m4 @@ -533,6 +533,7 @@ else O2IBCPPFLAGS="-I$O2IBPATH/include" EXTRA_KCFLAGS_save="$EXTRA_KCFLAGS" EXTRA_KCFLAGS="$EXTRA_KCFLAGS $O2IBCPPFLAGS" + EXTRA_LNET_INCLUDE="$O2IBCPPFLAGS $EXTRA_LNET_INCLUDE" LB_LINUX_TRY_COMPILE([ #include #if !HAVE_GFP_T @@ -566,8 +567,21 @@ else O2IBLND="" O2IBCPPFLAGS="" ]) + + IB_DMA_MAP="`grep -c ib_dma_map_single ${O2IBPATH}/include/rdma/ib_verbs.h`" + if test "$IB_DMA_MAP" != 0 ; then + IBLND_OFED_VERSION="102" + else + IBLND_OFED_VERSION="101" + fi + + AC_DEFINE_UNQUOTED(IBLND_OFED_VERSION, $IBLND_OFED_VERSION, + [OFED version]) + EXTRA_KCFLAGS="$EXTRA_KCFLAGS_save" fi + +AC_SUBST(EXTRA_LNET_INCLUDE) AC_SUBST(O2IBCPPFLAGS) AC_SUBST(O2IBLND) ]) diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c index ded32d6..226d453 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.c +++ b/lnet/klnds/o2iblnd/o2iblnd.c @@ -697,11 +697,10 @@ kiblnd_create_conn (kib_peer_t *peer, struct rdma_cm_id *cmid, int state) rx->rx_conn = conn; rx->rx_msg = (kib_msg_t *)(((char *)page_address(page)) + page_offset); - rx->rx_msgaddr = dma_map_single(cmid->device->dma_device, - rx->rx_msg, - IBLND_MSG_SIZE, - DMA_FROM_DEVICE); - pci_unmap_addr_set(rx, rx_msgunmap, rx->rx_msgaddr); + rx->rx_msgaddr = kiblnd_dma_map_single(cmid->device, + rx->rx_msg, IBLND_MSG_SIZE, + DMA_FROM_DEVICE); + KIBLND_UNMAP_ADDR_SET(rx, rx_msgunmap, rx->rx_msgaddr); CDEBUG(D_NET,"rx %d: %p "LPX64"("LPX64")\n", i, rx->rx_msg, rx->rx_msgaddr, @@ -882,9 +881,10 @@ kiblnd_destroy_conn (kib_conn_t *conn) LASSERT (rx->rx_nob >= 0); /* not posted */ - dma_unmap_single(conn->ibc_cmid->device->dma_device, - pci_unmap_addr(rx, rx_msgunmap), - IBLND_MSG_SIZE, DMA_FROM_DEVICE); + kiblnd_dma_unmap_single(conn->ibc_cmid->device, + KIBLND_UNMAP_ADDR(rx, rx_msgunmap, + rx->rx_msgaddr), + IBLND_MSG_SIZE, DMA_FROM_DEVICE); } kiblnd_free_pages(conn->ibc_rx_pages); @@ -1215,9 +1215,10 @@ kiblnd_unmap_tx_descs (lnet_ni_t *ni) for (i = 0; i < IBLND_TX_MSGS(); i++) { tx = &net->ibn_tx_descs[i]; - dma_unmap_single(net->ibn_dev->ibd_cmid->device->dma_device, - pci_unmap_addr(tx, tx_msgunmap), - IBLND_MSG_SIZE, DMA_TO_DEVICE); + kiblnd_dma_unmap_single(net->ibn_dev->ibd_cmid->device, + KIBLND_UNMAP_ADDR(tx, tx_msgunmap, + tx->tx_msgaddr), + IBLND_MSG_SIZE, DMA_TO_DEVICE); } } @@ -1246,10 +1247,10 @@ kiblnd_map_tx_descs (lnet_ni_t *ni) tx->tx_msg = (kib_msg_t *)(((char *)page_address(page)) + page_offset); - tx->tx_msgaddr = dma_map_single( - net->ibn_dev->ibd_cmid->device->dma_device, + tx->tx_msgaddr = kiblnd_dma_map_single( + net->ibn_dev->ibd_cmid->device, tx->tx_msg, IBLND_MSG_SIZE, DMA_TO_DEVICE); - pci_unmap_addr_set(tx, tx_msgunmap, tx->tx_msgaddr); + KIBLND_UNMAP_ADDR_SET(tx, tx_msgunmap, tx->tx_msgaddr); list_add(&tx->tx_list, &net->ibn_idle_txs); diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h index 8afba88..e27f860 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.h +++ b/lnet/klnds/o2iblnd/o2iblnd.h @@ -569,6 +569,110 @@ kiblnd_rd_size (kib_rdma_desc_t *rd) } #endif +#if (IBLND_OFED_VERSION == 102) + +static inline __u64 kiblnd_dma_map_single(struct ib_device *dev, + void *msg, size_t size, + enum dma_data_direction direction) +{ + return ib_dma_map_single(dev, msg, size, direction); +} + +static inline void kiblnd_dma_unmap_single(struct ib_device *dev, + __u64 addr, size_t size, + enum dma_data_direction direction) +{ + ib_dma_unmap_single(dev, addr, size, direction); +} + +#define KIBLND_UNMAP_ADDR_SET(p, m, a) do {} while (0) +#define KIBLND_UNMAP_ADDR(p, m, a) (a) + +static inline int kiblnd_dma_map_sg(struct ib_device *dev, + struct scatterlist *sg, int nents, + enum dma_data_direction direction) +{ + return ib_dma_map_sg(dev, sg, nents, direction); +} + +static inline void kiblnd_dma_unmap_sg(struct ib_device *dev, + struct scatterlist *sg, int nents, + enum dma_data_direction direction) +{ + ib_dma_unmap_sg(dev, sg, nents, direction); +} + +static inline __u64 kiblnd_sg_dma_address(struct ib_device *dev, + struct scatterlist *sg) +{ + return ib_sg_dma_address(dev, sg); +} + +static inline unsigned int kiblnd_sg_dma_len(struct ib_device *dev, + struct scatterlist *sg) +{ + return ib_sg_dma_len(dev, sg); +} + +/* XXX We use KIBLND_CONN_PARAM(e) as writable buffer, it's not strictly + * right because OFED1.2 defines it as const, to use it we have to add + * (void *) cast to overcome "const" */ + +#define KIBLND_CONN_PARAM(e) ((e)->param.conn.private_data) +#define KIBLND_CONN_PARAM_LEN(e) ((e)->param.conn.private_data_len) + +#elif (IBLND_OFED_VERSION == 101) + +static inline dma_addr_t kiblnd_dma_map_single(struct ib_device *dev, + void *msg, size_t size, + enum dma_data_direction direction) +{ + return dma_map_single(dev->dma_device, msg, size, direction); +} + +static inline void kiblnd_dma_unmap_single(struct ib_device *dev, + dma_addr_t addr, size_t size, + enum dma_data_direction direction) +{ + dma_unmap_single(dev->dma_device, addr, size, direction); +} + +#define KIBLND_UNMAP_ADDR_SET(p, m, a) pci_unmap_addr_set(p, m, a) +#define KIBLND_UNMAP_ADDR(p, m, a) pci_unmap_addr(p, m) + +static inline int kiblnd_dma_map_sg(struct ib_device *dev, + struct scatterlist *sg, int nents, + enum dma_data_direction direction) +{ + return dma_map_sg(dev->dma_device, sg, nents, direction); +} + +static inline void kiblnd_dma_unmap_sg(struct ib_device *dev, + struct scatterlist *sg, int nents, + enum dma_data_direction direction) +{ + return dma_unmap_sg(dev->dma_device, sg, nents, direction); +} + + +static inline dma_addr_t kiblnd_sg_dma_address(struct ib_device *dev, + struct scatterlist *sg) +{ + return sg_dma_address(sg); +} + + +static inline unsigned int kiblnd_sg_dma_len(struct ib_device *dev, + struct scatterlist *sg) +{ + return sg_dma_len(sg); +} + +#define KIBLND_CONN_PARAM(e) ((e)->private_data) +#define KIBLND_CONN_PARAM_LEN(e) ((e)->private_data_len) + +#endif + int kiblnd_startup (lnet_ni_t *ni); void kiblnd_shutdown (lnet_ni_t *ni); int kiblnd_ctl (lnet_ni_t *ni, unsigned int cmd, void *arg); diff --git a/lnet/klnds/o2iblnd/o2iblnd_cb.c b/lnet/klnds/o2iblnd/o2iblnd_cb.c index 74bcb94..e4c3716 100644 --- a/lnet/klnds/o2iblnd/o2iblnd_cb.c +++ b/lnet/klnds/o2iblnd/o2iblnd_cb.c @@ -90,8 +90,8 @@ kiblnd_tx_done (lnet_ni_t *ni, kib_tx_t *tx) } #else if (tx->tx_nfrags != 0) { - dma_unmap_sg(net->ibn_dev->ibd_cmid->device->dma_device, - tx->tx_frags, tx->tx_nfrags, tx->tx_dmadir); + kiblnd_dma_unmap_sg(net->ibn_dev->ibd_cmid->device, + tx->tx_frags, tx->tx_nfrags, tx->tx_dmadir); tx->tx_nfrags = 0; } #endif @@ -639,14 +639,17 @@ kiblnd_setup_rd_iov(lnet_ni_t *ni, kib_tx_t *tx, kib_rdma_desc_t *rd, tx->tx_nfrags = sg - tx->tx_frags; tx->tx_dmadir = (rd != tx->tx_rd) ? DMA_FROM_DEVICE : DMA_TO_DEVICE; - rd->rd_nfrags = dma_map_sg(net->ibn_dev->ibd_cmid->device->dma_device, - tx->tx_frags, tx->tx_nfrags, tx->tx_dmadir); + rd->rd_nfrags = kiblnd_dma_map_sg(net->ibn_dev->ibd_cmid->device, + tx->tx_frags, tx->tx_nfrags, + tx->tx_dmadir); rd->rd_key = (rd != tx->tx_rd) ? net->ibn_dev->ibd_mr->rkey : net->ibn_dev->ibd_mr->lkey; for (i = 0; i < rd->rd_nfrags; i++) { - rd->rd_frags[i].rf_nob = sg_dma_len(&tx->tx_frags[i]); - rd->rd_frags[i].rf_addr = sg_dma_address(&tx->tx_frags[i]); + rd->rd_frags[i].rf_nob = kiblnd_sg_dma_len( + net->ibn_dev->ibd_cmid->device, &tx->tx_frags[i]); + rd->rd_frags[i].rf_addr = kiblnd_sg_dma_address( + net->ibn_dev->ibd_cmid->device, &tx->tx_frags[i]); } return 0; @@ -697,14 +700,16 @@ kiblnd_setup_rd_kiov (lnet_ni_t *ni, kib_tx_t *tx, kib_rdma_desc_t *rd, tx->tx_nfrags = sg - tx->tx_frags; tx->tx_dmadir = (rd != tx->tx_rd) ? DMA_FROM_DEVICE : DMA_TO_DEVICE; - rd->rd_nfrags = dma_map_sg(net->ibn_dev->ibd_cmid->device->dma_device, - tx->tx_frags, tx->tx_nfrags, tx->tx_dmadir); + rd->rd_nfrags = kiblnd_dma_map_sg(net->ibn_dev->ibd_cmid->device, + tx->tx_frags, tx->tx_nfrags, tx->tx_dmadir); rd->rd_key = (rd != tx->tx_rd) ? net->ibn_dev->ibd_mr->rkey : net->ibn_dev->ibd_mr->lkey; for (i = 0; i < tx->tx_nfrags; i++) { - rd->rd_frags[i].rf_nob = sg_dma_len(&tx->tx_frags[i]); - rd->rd_frags[i].rf_addr = sg_dma_address(&tx->tx_frags[i]); + rd->rd_frags[i].rf_nob = kiblnd_sg_dma_len( + net->ibn_dev->ibd_cmid->device, &tx->tx_frags[i]); + rd->rd_frags[i].rf_addr = kiblnd_sg_dma_address( + net->ibn_dev->ibd_cmid->device, &tx->tx_frags[i]); #if 0 CDEBUG(D_WARNING,"frag[%d]: "LPX64" for %d\n", i, rd->rd_frags[i].rf_addr, rd->rd_frags[i].rf_nob); @@ -2610,8 +2615,8 @@ kiblnd_cm_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event) case RDMA_CM_EVENT_CONNECT_REQUEST: /* destroy cmid on failure */ rc = kiblnd_passive_connect(cmid, - event->private_data, - event->private_data_len); + (void *)KIBLND_CONN_PARAM(event), + KIBLND_CONN_PARAM_LEN(event)); CDEBUG(D_NET, "connreq: %d\n", rc); return rc; @@ -2703,8 +2708,8 @@ kiblnd_cm_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event) case IBLND_CONN_ACTIVE_CONNECT: kiblnd_rejected(conn, event->status, - event->private_data, - event->private_data_len); + (void *)KIBLND_CONN_PARAM(event), + KIBLND_CONN_PARAM_LEN(event)); break; } kiblnd_conn_decref(conn); @@ -2726,8 +2731,8 @@ kiblnd_cm_callback(struct rdma_cm_id *cmid, struct rdma_cm_event *event) CDEBUG(D_NET, "ESTABLISHED(active): %s\n", libcfs_nid2str(conn->ibc_peer->ibp_nid)); kiblnd_check_connreply(conn, - event->private_data, - event->private_data_len); + (void *)KIBLND_CONN_PARAM(event), + KIBLND_CONN_PARAM_LEN(event)); break; } /* net keeps its ref on conn! */