From cc5594df3e70d1924f34ccdf4c3178654d277ad0 Mon Sep 17 00:00:00 2001 From: Shaun Tancheff Date: Sun, 23 Apr 2023 07:19:11 -0500 Subject: [PATCH] LU-16759 o2ib: MOFED 5.5+ ib_dma_virt_map_sg MOFED 5.5 fails with: ERROR: "ib_dma_virt_map_sg" [.../ko2iblnd.ko] undefined! See if we have a broken ib_dma_map_sg() and provide a suitable replacement for the missing functionality. Test-Parameters: trivial HPE-bug-id: LUS-11587 Signed-off-by: Shaun Tancheff Change-Id: I3b4454fcafe4640c15b13385a0209ed71f51a3d0 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/50711 Tested-by: jenkins Tested-by: Maloo Reviewed-by: xinliang Reviewed-by: Petros Koutoupis Reviewed-by: Jian Yu Reviewed-by: Oleg Drokin --- lnet/autoconf/lustre-lnet.m4 | 30 +++++++++++++++++++++++++++++- lnet/klnds/o2iblnd/o2iblnd.h | 24 +++++++++++++++++++++--- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4 index 26c2a0d..625f797 100644 --- a/lnet/autoconf/lustre-lnet.m4 +++ b/lnet/autoconf/lustre-lnet.m4 @@ -246,7 +246,7 @@ AS_IF([test $ENABLEO2IB = "no"], [ O2IB_SYMVER=$LINUX_OBJ/Module.symvers fi if test -n "$O2IB_SYMVER"; then - if test ！"$O2IB_SYMVER" -ef "$LINUX_OBJ/Module.symvers"; then + if test ! "$O2IB_SYMVER" -ef "$LINUX_OBJ/Module.symvers"; then AC_MSG_NOTICE([adding $O2IB_SYMVER to Symbol Path O2IB]) EXTRA_SYMBOLS="$EXTRA_SYMBOLS $O2IB_SYMVER" AC_SUBST(EXTRA_SYMBOLS) @@ -612,6 +612,34 @@ AS_IF([test $ENABLEO2IB != "no"], [ ]) ]) + # MOFED 5.5 fails with: + # ERROR: "ib_dma_virt_map_sg" [.../ko2iblnd.ko] undefined! + # See if we have a broken ib_dma_map_sg() + AC_DEFUN([LN_SRC_SANE_IB_DMA_MAP_SG], [ + LB2_LINUX_TEST_SRC([sane_ib_dma_map_sg], [ + #ifdef HAVE_COMPAT_RDMA + #undef PACKAGE_NAME + #undef PACKAGE_TARNAME + #undef PACKAGE_VERSION + #undef PACKAGE_STRING + #undef PACKAGE_BUGREPORT + #undef PACKAGE_URL + #include + #endif + #include + ],[ + ib_dma_map_sg((struct ib_device *)NULL, + (struct scatterlist *)NULL, 1, 0); + ],[-Werror],[$EXTRA_OFED_CONFIG $EXTRA_OFED_INCLUDE]) + ]) + AC_DEFUN([LN_SANE_IB_DMA_MAP_SG], [ + AC_MSG_CHECKING([if ib_dma_map_sg() is sane]) + LB2_LINUX_TEST_RESULT([sane_ib_dma_map_sg], [ + AC_DEFINE(HAVE_SANE_IB_DMA_MAP_SG, 1, + [ib_dma_map_sg is sane]) + ]) + ]) + # # LN_IB_DEVICE_OPS_EXISTS # diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h index 396a708..f7936dc 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.h +++ b/lnet/klnds/o2iblnd/o2iblnd.h @@ -1078,15 +1078,33 @@ static inline void kiblnd_dma_unmap_single(struct ib_device *dev, static inline int kiblnd_dma_map_sg(struct kib_hca_dev *hdev, struct kib_tx *tx) { + struct ib_device *dev = hdev->ibh_ibdev; struct scatterlist *sg = tx->tx_frags; int nents = tx->tx_nfrags; enum dma_data_direction direction = tx->tx_dmadir; if (tx->tx_gpu) - return lnet_rdma_map_sg_attrs(hdev->ibh_ibdev->dma_device, - sg, nents, direction); + return lnet_rdma_map_sg_attrs(dev->dma_device, sg, nents, + direction); - return ib_dma_map_sg(hdev->ibh_ibdev, sg, nents, direction); +#ifdef HAVE_SANE_IB_DMA_MAP_SG + return ib_dma_map_sg(dev, sg, nents, direction); +#else + #ifdef CONFIG_INFINIBAND_VIRT_DMA + if (!dev->dma_device) { + struct scatterlist *s; + int i; + + /* NOTE: open coded ib_dma_virt_map_sg() */ + for_each_sg(sg, s, nents, i) { + sg_dma_address(s) = (uintptr_t)sg_virt(s); + sg_dma_len(s) = s->length; + } + return nents; + } + #endif + return dma_map_sg_attrs(dev->dma_device, sg, nents, direction, 0); +#endif } static inline -- 1.8.3.1