From 04157900fafdf8c4b9ec40d5bf7ba306789509ed Mon Sep 17 00:00:00 2001 From: Timothy Day Date: Tue, 13 May 2025 21:37:17 +0000 Subject: [PATCH] LU-19013 lnet: fix wording for GDS configure check The wording on the GDS/CUDA configuration options is incorrect. If the user does not specify external headers, Lustre will fallback to the embedded headers rather than disabling GDS. Fix the wording on the configure options, improve the macro name, and reorganize the header such that correct defintions are under the ifdef. Fixes: c65eabc2b113 ("LU-15189 build: add GDS configure options") Test-Parameters: trivial Signed-off-by: Timothy Day Change-Id: I645bc1c0c4bf26bdb9841c849b6cf8eebdc0bdee Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/59215 Reviewed-by: Andreas Dilger Reviewed-by: Alexey Lyashkov Reviewed-by: Oleg Drokin Tested-by: jenkins Tested-by: Maloo --- config/lustre-lnet.m4 | 8 ++--- lnet/include/lnet/Makefile.am | 1 - lnet/include/lnet/lnet_gds.h | 59 ------------------------------------- lnet/include/lnet/lnet_rdma.h | 68 ++++++++++++++++++++++++++++++++++++++----- lnet/lnet/lnet_rdma.c | 9 ------ 5 files changed, 64 insertions(+), 81 deletions(-) delete mode 100644 lnet/include/lnet/lnet_gds.h diff --git a/config/lustre-lnet.m4 b/config/lustre-lnet.m4 index bc791e8..cb2a813 100644 --- a/config/lustre-lnet.m4 +++ b/config/lustre-lnet.m4 @@ -1322,13 +1322,13 @@ AC_MSG_NOTICE([LNet core checks AC_ARG_WITH([cuda], AS_HELP_STRING([--with-cuda=path], - [Path to the CUDA sources. Set to 'no' to disable.]), + [Path to the CUDA sources.]), [cudapath="$withval"], [cudapath1=`ls -d1 /usr/src/nvidia-*/nvidia/ 2>/dev/null | tail -1`]) AC_ARG_WITH([gds], AS_HELP_STRING([--with-gds=path], - [Path to the GDS sources. Set to 'no' to disable.]), + [Path to the GDS sources.]), [gdspath="$withval"], [gdspath1=`ls -d1 /usr/src/nvidia-fs*/ 2>/dev/null | tail -1`]) @@ -1366,7 +1366,7 @@ AS_IF([test -e "${cudapath}" && test -e "${gdspath}"],[ AC_MSG_RESULT([GDS path is ${gdspath}]) [GDS_PATH=${gdspath}] AC_SUBST(GDS_PATH) - AC_DEFINE(WITH_GDS, 1, "GDS build enabled") + AC_DEFINE(WITH_EXTERNAL_GDS_HEADER, 1, "GDS build enabled") ], [ AC_MSG_RESULT([GDS sources not found: config-host.h does not exist]) ]) @@ -1374,7 +1374,7 @@ AS_IF([test -e "${cudapath}" && test -e "${gdspath}"],[ AC_MSG_RESULT([GDS sources not found: nvfs-dma.h does not exist]) ]) ],[ - AC_MSG_WARN([CUDA or GDS sources not found. GDS support disabled]) + AC_MSG_WARN([CUDA or GDS sources not found. Using embedded GDS headers.]) ]) # lnet/utils/lnetconfig/liblnetconfig_netlink.c diff --git a/lnet/include/lnet/Makefile.am b/lnet/include/lnet/Makefile.am index b9508da..f4d629c 100644 --- a/lnet/include/lnet/Makefile.am +++ b/lnet/include/lnet/Makefile.am @@ -13,6 +13,5 @@ EXTRA_DIST = \ lnet_compat.h \ lnet_crypto.h \ lnet_rdma.h \ - lnet_gds.h \ lock.h \ socklnd.h diff --git a/lnet/include/lnet/lnet_gds.h b/lnet/include/lnet/lnet_gds.h deleted file mode 100644 index 0b5d410..0000000 --- a/lnet/include/lnet/lnet_gds.h +++ /dev/null @@ -1,59 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -/* This file is part of Lustre, http://www.lustre.org/ */ - -#ifndef LUSTRE_NVFS_H -#define LUSTRE_NVFS_H - -#include -#include -#include -#include -#include -#include -#include -#include - - -struct nvfs_dma_rw_ops { - unsigned long long ft_bmap; /* feature bitmap */ - - int (*nvfs_blk_rq_map_sg) (struct request_queue *q, - struct request *req, - struct scatterlist *sglist); - - int (*nvfs_dma_map_sg_attrs) (struct device *device, - struct scatterlist *sglist, - int nents, - enum dma_data_direction dma_dir, - unsigned long attrs); - - int (*nvfs_dma_unmap_sg) (struct device *device, - struct scatterlist *sglist, - int nents, - enum dma_data_direction dma_dir); - bool (*nvfs_is_gpu_page) (struct page *); - unsigned int (*nvfs_gpu_index) (struct page *page); - unsigned int (*nvfs_device_priority) (struct device *dev, unsigned int dev_index); -}; - -/* feature list for dma_ops, values indicate bit pos */ -enum ft_bits { - nvfs_ft_prep_sglist = 1ULL << 0, - nvfs_ft_map_sglist = 1ULL << 1, - nvfs_ft_is_gpu_page = 1ULL << 2, - nvfs_ft_device_priority = 1ULL << 3, -}; - -/* check features for use in registration with vendor drivers */ -#define NVIDIA_FS_CHECK_FT_SGLIST_PREP(ops) \ - ((ops)->ft_bmap & nvfs_ft_prep_sglist) -#define NVIDIA_FS_CHECK_FT_SGLIST_DMA(ops) \ - ((ops)->ft_bmap & nvfs_ft_map_sglist) -#define NVIDIA_FS_CHECK_FT_GPU_PAGE(ops) \ - ((ops)->ft_bmap & nvfs_ft_is_gpu_page) -#define NVIDIA_FS_CHECK_FT_DEVICE_PRIORITY(ops) \ - ((ops)->ft_bmap & nvfs_ft_device_priority) - -#endif /* LUSTRE_NVFS_H */ - diff --git a/lnet/include/lnet/lnet_rdma.h b/lnet/include/lnet/lnet_rdma.h index a21d775..fbec4c7 100644 --- a/lnet/include/lnet/lnet_rdma.h +++ b/lnet/include/lnet/lnet_rdma.h @@ -5,6 +5,65 @@ #ifndef LNET_RDMA_H #define LNET_RDMA_H +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef WITH_EXTERNAL_GDS_HEADER +#include "nvfs-dma.h" +#else + +struct nvfs_dma_rw_ops { + unsigned long long ft_bmap; /* feature bitmap */ + + int (*nvfs_blk_rq_map_sg)(struct request_queue *q, + struct request *req, + struct scatterlist *sglist); + + int (*nvfs_dma_map_sg_attrs)(struct device *device, + struct scatterlist *sglist, + int nents, + enum dma_data_direction dma_dir, + unsigned long attrs); + + int (*nvfs_dma_unmap_sg)(struct device *device, + struct scatterlist *sglist, + int nents, + enum dma_data_direction dma_dir); + bool (*nvfs_is_gpu_page)(struct page *page); + unsigned int (*nvfs_gpu_index)(struct page *page); + unsigned int (*nvfs_device_priority)(struct device *dev, + unsigned int dev_index); +}; + +/* feature list for dma_ops, values indicate bit pos */ +enum ft_bits { + nvfs_ft_prep_sglist = 1ULL << 0, + nvfs_ft_map_sglist = 1ULL << 1, + nvfs_ft_is_gpu_page = 1ULL << 2, + nvfs_ft_device_priority = 1ULL << 3, +}; + +/* check features for use in registration with vendor drivers */ +#define NVIDIA_FS_CHECK_FT_SGLIST_PREP(ops) \ + ((ops)->ft_bmap & nvfs_ft_prep_sglist) +#define NVIDIA_FS_CHECK_FT_SGLIST_DMA(ops) \ + ((ops)->ft_bmap & nvfs_ft_map_sglist) +#define NVIDIA_FS_CHECK_FT_GPU_PAGE(ops) \ + ((ops)->ft_bmap & nvfs_ft_is_gpu_page) +#define NVIDIA_FS_CHECK_FT_DEVICE_PRIORITY(ops) \ + ((ops)->ft_bmap & nvfs_ft_device_priority) + +#define NVFS_IO_ERR -1 +#define NVFS_CPU_REQ -2 + +#endif /* WITH_EXTERNEL_GDS_HEADER */ + #define REGSTR2(x) x##_register_nvfs_dma_ops #define REGSTR(x) REGSTR2(x) @@ -16,13 +75,6 @@ #define REGISTER_FUNC REGSTR(MODULE_PREFIX) #define UNREGISTER_FUNC UNREGSTR(MODULE_PREFIX) -struct device; -struct page; -enum dma_data_direction; -struct scatterlist; - -struct nvfs_dma_rw_ops; - int REGISTER_FUNC(struct nvfs_dma_rw_ops *ops); void UNREGISTER_FUNC(void); @@ -41,5 +93,5 @@ unsigned int lnet_get_dev_idx(struct page *page); #define DMA_ATTR_NO_WARN 0 #endif -#endif /* LUSTRE_NVFS_H */ +#endif /* LNET_RDMA_H */ diff --git a/lnet/lnet/lnet_rdma.c b/lnet/lnet/lnet_rdma.c index b0eaad4..dd72432 100644 --- a/lnet/lnet/lnet_rdma.c +++ b/lnet/lnet/lnet_rdma.c @@ -2,21 +2,12 @@ /* This file is part of Lustre, http://www.lustre.org/ */ -#ifdef WITH_GDS -#include "nvfs-dma.h" -#else -#include -#endif - #include #include /* MAX / MIN conflict */ #include -#define NVFS_IO_ERR -1 -#define NVFS_CPU_REQ -2 - #define NVFS_HOLD_TIME_MS 1000 #define ERROR_PRINT_DEADLINE 3600 -- 1.8.3.1