lnet/lnet/lnet_rdma.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /* This file is part of Lustre, http://www.lustre.org/ */
   4
   5 #ifdef WITH_GDS
   6 #include "nvfs-dma.h"
   7 #else
   8 #include <lnet/lnet_gds.h>
   9 #endif
  10
  11 #include <lnet/lnet_rdma.h>
  12 #include <libcfs/libcfs.h>
  13
  14 /* MAX / MIN conflict */
  15 #include <lnet/lib-lnet.h>
  16
  17 #define NVFS_IO_ERR                     -1
  18 #define NVFS_CPU_REQ                    -2
  19
  20 #define NVFS_HOLD_TIME_MS 1000
  21
  22 #define ERROR_PRINT_DEADLINE 3600
  23
  24 atomic_t nvfs_shutdown = ATOMIC_INIT(1);
  25 struct nvfs_dma_rw_ops *nvfs_ops = NULL;
  26 struct percpu_counter nvfs_n_ops;
  27
  28 static inline long nvfs_count_ops(void)
  29 {
  30         return percpu_counter_sum(&nvfs_n_ops);
  31 }
  32
  33 static struct nvfs_dma_rw_ops *nvfs_get_ops(void)
  34 {
  35         if (!nvfs_ops || atomic_read(&nvfs_shutdown))
  36                 return NULL;
  37
  38         percpu_counter_inc(&nvfs_n_ops);
  39
  40         return nvfs_ops;
  41 }
  42
  43 static inline void nvfs_put_ops(void)
  44 {
  45         percpu_counter_dec(&nvfs_n_ops);
  46 }
  47
  48 static inline bool nvfs_check_feature_set(struct nvfs_dma_rw_ops *ops)
  49 {
  50         bool supported = true;
  51         static time64_t last_printed;
  52
  53         if (unlikely(!NVIDIA_FS_CHECK_FT_SGLIST_PREP(ops))) {
  54                 if ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)
  55                         CDEBUG(D_CONSOLE,
  56                                "NVFS sg list preparation callback missing\n");
  57                 supported = false;
  58         }
  59         if (unlikely(!NVIDIA_FS_CHECK_FT_SGLIST_DMA(ops))) {
  60                 if ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)
  61                         CDEBUG(D_CONSOLE,
  62                                "NVFS DMA mapping callbacks missing\n");
  63                 supported = false;
  64         }
  65         if (unlikely(!NVIDIA_FS_CHECK_FT_GPU_PAGE(ops))) {
  66                 if ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)
  67                         CDEBUG(D_CONSOLE,
  68                                "NVFS page identification callback missing\n");
  69                 supported = false;
  70         }
  71         if (unlikely(!NVIDIA_FS_CHECK_FT_DEVICE_PRIORITY(ops))) {
  72                 if ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)
  73                         CDEBUG(D_CONSOLE,
  74                                "NVFS device priority callback not missing\n");
  75                 supported = false;
  76         }
  77
  78         if (unlikely(!supported &&
  79                      ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)))
  80                 last_printed = ktime_get_seconds();
  81         else if (supported)
  82                 last_printed = 0;
  83
  84         return supported;
  85 }
  86
  87 int REGISTER_FUNC(struct nvfs_dma_rw_ops *ops)
  88 {
  89         if (!ops || !nvfs_check_feature_set(ops))
  90                 return -EINVAL;
  91
  92         nvfs_ops = ops;
  93         (void)percpu_counter_init(&nvfs_n_ops, 0, GFP_KERNEL);
  94         atomic_set(&nvfs_shutdown, 0);
  95         CDEBUG(D_NET, "registering nvfs %p\n", ops);
  96         return 0;
  97 }
  98 EXPORT_SYMBOL(REGISTER_FUNC);
  99
 100 void UNREGISTER_FUNC(void)
 101 {
 102         (void)atomic_cmpxchg(&nvfs_shutdown, 0, 1);
 103         do {
 104                 CDEBUG(D_NET, "Attempting to de-register nvfs: %ld\n",
 105                        nvfs_count_ops());
 106                 msleep(NVFS_HOLD_TIME_MS);
 107         } while (nvfs_count_ops());
 108         nvfs_ops = NULL;
 109         percpu_counter_destroy(&nvfs_n_ops);
 110 }
 111 EXPORT_SYMBOL(UNREGISTER_FUNC);
 112
 113 unsigned int
 114 lnet_get_dev_prio(struct device *dev, unsigned int dev_idx)
 115 {
 116         unsigned int dev_prio = UINT_MAX;
 117         struct nvfs_dma_rw_ops *nvfs_ops;
 118
 119         if (!dev)
 120                 return dev_prio;
 121
 122         nvfs_ops = nvfs_get_ops();
 123         if (!nvfs_ops)
 124                 return dev_prio;
 125
 126         dev_prio = nvfs_ops->nvfs_device_priority (dev, dev_idx);
 127
 128         nvfs_put_ops();
 129         return dev_prio;
 130 }
 131 EXPORT_SYMBOL(lnet_get_dev_prio);
 132
 133 unsigned int
 134 lnet_get_dev_idx(struct page *page)
 135 {
 136         unsigned int dev_idx = UINT_MAX;
 137         struct nvfs_dma_rw_ops *nvfs_ops;
 138
 139         nvfs_ops = nvfs_get_ops();
 140         if (!nvfs_ops)
 141                 return dev_idx;
 142
 143         dev_idx = nvfs_ops->nvfs_gpu_index(page);
 144
 145         nvfs_put_ops();
 146         return dev_idx;
 147 }
 148
 149 int lnet_rdma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
 150                            int nents, enum dma_data_direction direction)
 151 {
 152         struct nvfs_dma_rw_ops *nvfs_ops = nvfs_get_ops();
 153
 154         if (nvfs_ops) {
 155                 int count;
 156
 157                 count = nvfs_ops->nvfs_dma_map_sg_attrs(dev,
 158                                 sg, nents, direction,
 159                                 DMA_ATTR_NO_WARN);
 160
 161                 if (unlikely((count == NVFS_IO_ERR))) {
 162                         nvfs_put_ops();
 163                         return -EIO;
 164                 }
 165
 166                 if (unlikely(count == NVFS_CPU_REQ))
 167                         nvfs_put_ops();
 168                 else
 169                         return count;
 170         }
 171
 172         return 0;
 173 }
 174 EXPORT_SYMBOL(lnet_rdma_map_sg_attrs);
 175
 176 int lnet_rdma_unmap_sg(struct device *dev,
 177                        struct scatterlist *sg, int nents,
 178                        enum dma_data_direction direction)
 179 {
 180         struct nvfs_dma_rw_ops *nvfs_ops = nvfs_get_ops();
 181
 182         if (nvfs_ops) {
 183                 int count;
 184
 185                 count = nvfs_ops->nvfs_dma_unmap_sg(dev, sg,
 186                                                     nents, direction);
 187
 188                 /* drop the count we got by calling nvfs_get_ops() */
 189                 nvfs_put_ops();
 190
 191                 if (count) {
 192                         nvfs_put_ops();
 193                         return count;
 194                 }
 195         }
 196
 197         return 0;
 198 }
 199 EXPORT_SYMBOL(lnet_rdma_unmap_sg);
 200
 201 bool
 202 lnet_is_rdma_only_page(struct page *page)
 203 {
 204         bool is_gpu_page = false;
 205         struct nvfs_dma_rw_ops *nvfs_ops;
 206
 207         LASSERT(page != NULL);
 208
 209         nvfs_ops = nvfs_get_ops();
 210         if (nvfs_ops != NULL) {
 211                 is_gpu_page = nvfs_ops->nvfs_is_gpu_page(page);
 212                 nvfs_put_ops();
 213         }
 214         return is_gpu_page;
 215 }
 216 EXPORT_SYMBOL(lnet_is_rdma_only_page);