From: Alexey Lyashkov Date: Thu, 3 Mar 2022 16:52:41 +0000 (+0300) Subject: LU-15189 build: add GDS configure options X-Git-Tag: 2.15.51~242 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=refs%2Fchanges%2F80%2F45480%2F10;p=fs%2Flustre-release.git LU-15189 build: add GDS configure options add ability to specify GDS patch, pickup an GDS API directly from the GDS sources. Test-Parameters: trivial Signed-off-by: Alexey Lyashkov Change-Id: Iccc52765540cc90c6a69af34d230ed24b2eb996a Reviewed-on: https://review.whamcloud.com/45480 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Alexander Zarochentsev Reviewed-by: Andrew Perepechko Reviewed-by: Oleg Drokin --- diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4 index daeb1cc..35232d0 100644 --- a/lnet/autoconf/lustre-lnet.m4 +++ b/lnet/autoconf/lustre-lnet.m4 @@ -927,6 +927,43 @@ AC_DEFUN([LN_CONFIGURE], [ AC_MSG_NOTICE([LNet core checks ==============================================================================]) +AC_ARG_WITH([cuda], + AC_HELP_STRING([--with-cuda=path], + [Use a CUDA sources.]), + [LB_ARG_CANON_PATH([cuda], [CUDA_PATH])], + [CUDA_PATH=`ls -d1 /usr/src/nvidia-*/nvidia/ | tail -1`] +) + +AC_ARG_WITH([gds], + AC_HELP_STRING([--with-gds=path], + [Use a gds sources.]), + [LB_ARG_CANON_PATH([gds], [GDS_PATH])], + [GDS_PATH=`ls -d1 /usr/src/nvidia-fs* | tail -1`] +) + +AS_IF([test -n "${CUDA_PATH}" && test -n "${GDS_PATH}"],[ +LB_CHECK_FILE([$CUDA_PATH/nv-p2p.h], + [ + AC_MSG_RESULT([CUDA path is $CUDA_PATH]) + AC_SUBST(CUDA_PATH) + ], + [AC_MSG_ERROR([CUDA sources don't found. nv-p2p.h don't exit])] +) + +LB_CHECK_FILE([$GDS_PATH/nvfs-dma.h], + [ + LB_CHECK_FILE([$GDS_PATH/config-host.h], [ + AC_MSG_RESULT([GDS path is $GDS_PATH]) + AC_SUBST(GDS_PATH) + AC_DEFINE(WITH_GDS, 1, "GDS build enabled") + ], []) + ], + []) +],[ + AC_MSG_WARN([CUDA or GDS sources don't found. GDS support disabled]) +] +) + # lnet/utils/lnetconfig/liblnetconfig_netlink.c AS_IF([test "x$enable_dist" = xno], [ PKG_CHECK_MODULES(LIBNL3, [libnl-genl-3.0 >= 3.1]) diff --git a/lnet/include/lnet/Makefile.am b/lnet/include/lnet/Makefile.am index 3f2294f..1eaa042 100644 --- a/lnet/include/lnet/Makefile.am +++ b/lnet/include/lnet/Makefile.am @@ -4,4 +4,5 @@ EXTRA_DIST = \ lib-types.h \ udsp.h \ lnet_rdma.h \ + lnet_gds.h \ socklnd.h diff --git a/lnet/include/lnet/lnet_gds.h b/lnet/include/lnet/lnet_gds.h new file mode 100644 index 0000000..f008125 --- /dev/null +++ b/lnet/include/lnet/lnet_gds.h @@ -0,0 +1,55 @@ +#ifndef LUSTRE_NVFS_H +#define LUSTRE_NVFS_H + +#include +#include +#include +#include +#include +#include +#include +#include + + +struct nvfs_dma_rw_ops { + unsigned long long ft_bmap; /* feature bitmap */ + + int (*nvfs_blk_rq_map_sg) (struct request_queue *q, + struct request *req, + struct scatterlist *sglist); + + int (*nvfs_dma_map_sg_attrs) (struct device *device, + struct scatterlist *sglist, + int nents, + enum dma_data_direction dma_dir, + unsigned long attrs); + + int (*nvfs_dma_unmap_sg) (struct device *device, + struct scatterlist *sglist, + int nents, + enum dma_data_direction dma_dir); + bool (*nvfs_is_gpu_page) (struct page *); + unsigned int (*nvfs_gpu_index) (struct page *page); + unsigned int (*nvfs_device_priority) (struct device *dev, unsigned int dev_index); +}; + +/* feature list for dma_ops, values indicate bit pos */ +enum ft_bits { + nvfs_ft_prep_sglist = 1ULL << 0, + nvfs_ft_map_sglist = 1ULL << 1, + nvfs_ft_is_gpu_page = 1ULL << 2, + nvfs_ft_device_priority = 1ULL << 3, +}; + +/* check features for use in registration with vendor drivers */ +#define NVIDIA_FS_CHECK_FT_SGLIST_PREP(ops) \ + ((ops)->ft_bmap & nvfs_ft_prep_sglist) +#define NVIDIA_FS_CHECK_FT_SGLIST_DMA(ops) \ + ((ops)->ft_bmap & nvfs_ft_map_sglist) +#define NVIDIA_FS_CHECK_FT_GPU_PAGE(ops) \ + ((ops)->ft_bmap & nvfs_ft_is_gpu_page) +#define NVIDIA_FS_CHECK_FT_DEVICE_PRIORITY(ops) \ + ((ops)->ft_bmap & nvfs_ft_device_priority) + +#endif /* LUSTRE_NVFS_H */ + diff --git a/lnet/include/lnet/lnet_rdma.h b/lnet/include/lnet/lnet_rdma.h index 6aa5367..94b1704 100644 --- a/lnet/include/lnet/lnet_rdma.h +++ b/lnet/include/lnet/lnet_rdma.h @@ -1,70 +1,12 @@ -#ifndef LUSTRE_NVFS_H -#define LUSTRE_NVFS_H +#ifndef LNET_RDMA_H +#define LNET_RDMA_H -#include -#include -#include -#include -#include -#include -#include -#include +struct device; +struct page; +enum dma_data_direction; +struct scatterlist; -#define REGSTR2(x) x##_register_nvfs_dma_ops -#define REGSTR(x) REGSTR2(x) - -#define UNREGSTR2(x) x##_unregister_nvfs_dma_ops -#define UNREGSTR(x) UNREGSTR2(x) - -#define MODULE_PREFIX lustre_v1 - -#define REGISTER_FUNC REGSTR(MODULE_PREFIX) -#define UNREGISTER_FUNC UNREGSTR(MODULE_PREFIX) - -#define NVFS_IO_ERR -1 -#define NVFS_CPU_REQ -2 - -#define NVFS_HOLD_TIME_MS 1000 - -struct nvfs_dma_rw_ops { - unsigned long long ft_bmap; /* feature bitmap */ - - int (*nvfs_blk_rq_map_sg) (struct request_queue *q, - struct request *req, - struct scatterlist *sglist); - - int (*nvfs_dma_map_sg_attrs) (struct device *device, - struct scatterlist *sglist, - int nents, - enum dma_data_direction dma_dir, - unsigned long attrs); - - int (*nvfs_dma_unmap_sg) (struct device *device, - struct scatterlist *sglist, - int nents, - enum dma_data_direction dma_dir); - bool (*nvfs_is_gpu_page) (struct page *); - unsigned int (*nvfs_gpu_index) (struct page *page); - unsigned int (*nvfs_device_priority) (struct device *dev, unsigned int dev_index); -}; - -/* feature list for dma_ops, values indicate bit pos */ -enum ft_bits { - nvfs_ft_prep_sglist = 1ULL << 0, - nvfs_ft_map_sglist = 1ULL << 1, - nvfs_ft_is_gpu_page = 1ULL << 2, - nvfs_ft_device_priority = 1ULL << 3, -}; - -/* check features for use in registration with vendor drivers */ -#define NVIDIA_FS_CHECK_FT_SGLIST_PREP(ops) \ - ((ops)->ft_bmap & nvfs_ft_prep_sglist) -#define NVIDIA_FS_CHECK_FT_SGLIST_DMA(ops) \ - ((ops)->ft_bmap & nvfs_ft_map_sglist) -#define NVIDIA_FS_CHECK_FT_GPU_PAGE(ops) \ - ((ops)->ft_bmap & nvfs_ft_is_gpu_page) -#define NVIDIA_FS_CHECK_FT_DEVICE_PRIORITY(ops) \ - ((ops)->ft_bmap & nvfs_ft_device_priority) +struct nvfs_dma_rw_ops; int REGISTER_FUNC (struct nvfs_dma_rw_ops *ops); diff --git a/lnet/lnet/Makefile.in b/lnet/lnet/Makefile.in index a85d6c1..96ebd61 100644 --- a/lnet/lnet/Makefile.in +++ b/lnet/lnet/Makefile.in @@ -5,6 +5,8 @@ lnet-objs += lib-me.o lib-msg.o lib-md.o lib-ptl.o lnet-objs += lib-socket.o lib-move.o module.o lo.o lnet-objs += router.o router_proc.o acceptor.o peer.o net_fault.o udsp.o +CFLAGS_lnet_rdma.o += -I @GDS_PATH@ -I@CUDA_PATH@ + default: all @INCLUDE_RULES@ diff --git a/lnet/lnet/lnet_rdma.c b/lnet/lnet/lnet_rdma.c index c5c9d9f..791ece04 100644 --- a/lnet/lnet/lnet_rdma.c +++ b/lnet/lnet/lnet_rdma.c @@ -1,7 +1,31 @@ +#ifdef WITH_GDS +#include "nvfs-dma.h" +#else +#include +#endif + #include #include + +/* MAX / MIN conflict */ #include +#define REGSTR2(x) x##_register_nvfs_dma_ops +#define REGSTR(x) REGSTR2(x) + +#define UNREGSTR2(x) x##_unregister_nvfs_dma_ops +#define UNREGSTR(x) UNREGSTR2(x) + +#define MODULE_PREFIX lustre_v1 + +#define REGISTER_FUNC REGSTR(MODULE_PREFIX) +#define UNREGISTER_FUNC UNREGSTR(MODULE_PREFIX) + +#define NVFS_IO_ERR -1 +#define NVFS_CPU_REQ -2 + +#define NVFS_HOLD_TIME_MS 1000 + #define ERROR_PRINT_DEADLINE 3600 atomic_t nvfs_shutdown = ATOMIC_INIT(1); @@ -113,6 +137,22 @@ lnet_get_dev_prio(struct device *dev, unsigned int dev_idx) } EXPORT_SYMBOL(lnet_get_dev_prio); +unsigned int +lnet_get_dev_idx(struct page *page) +{ + unsigned int dev_idx = UINT_MAX; + struct nvfs_dma_rw_ops *nvfs_ops; + + nvfs_ops = nvfs_get_ops(); + if (!nvfs_ops) + return dev_idx; + + dev_idx = nvfs_ops->nvfs_gpu_index(page); + + nvfs_put_ops(); + return dev_idx; +} + int lnet_rdma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction direction) { @@ -188,21 +228,3 @@ out: return found; } EXPORT_SYMBOL(lnet_is_rdma_only_page); - -unsigned int -lnet_get_dev_idx(struct page *page) -{ - unsigned int dev_idx = UINT_MAX; - struct nvfs_dma_rw_ops *nvfs_ops; - - nvfs_ops = nvfs_get_ops(); - if (!nvfs_ops) - return dev_idx; - - dev_idx = nvfs_ops->nvfs_gpu_index(page); - - nvfs_put_ops(); - return dev_idx; -} -EXPORT_SYMBOL(lnet_get_dev_idx); -