lnet/lnet/lnet_rdma.c

   1 #ifdef WITH_GDS
   2 #include "nvfs-dma.h"
   3 #else
   4 #include <lnet/lnet_gds.h>
   5 #endif
   6
   7 #include <lnet/lnet_rdma.h>
   8 #include <libcfs/libcfs.h>
   9
  10 /* MAX / MIN conflict */
  11 #include <lnet/lib-lnet.h>
  12
  13 #define REGSTR2(x) x##_register_nvfs_dma_ops
  14 #define REGSTR(x)  REGSTR2(x)
  15
  16 #define UNREGSTR2(x) x##_unregister_nvfs_dma_ops
  17 #define UNREGSTR(x)  UNREGSTR2(x)
  18
  19 #define MODULE_PREFIX lustre_v1
  20
  21 #define REGISTER_FUNC REGSTR(MODULE_PREFIX)
  22 #define UNREGISTER_FUNC UNREGSTR(MODULE_PREFIX)
  23
  24 #define NVFS_IO_ERR                     -1
  25 #define NVFS_CPU_REQ                    -2
  26
  27 #define NVFS_HOLD_TIME_MS 1000
  28
  29 #define ERROR_PRINT_DEADLINE 3600
  30
  31 atomic_t nvfs_shutdown = ATOMIC_INIT(1);
  32 struct nvfs_dma_rw_ops *nvfs_ops = NULL;
  33 struct percpu_counter nvfs_n_ops;
  34
  35 static inline long nvfs_count_ops(void)
  36 {
  37         return percpu_counter_sum(&nvfs_n_ops);
  38 }
  39
  40 static struct nvfs_dma_rw_ops *nvfs_get_ops(void)
  41 {
  42         if (!nvfs_ops || atomic_read(&nvfs_shutdown))
  43                 return NULL;
  44
  45         percpu_counter_inc(&nvfs_n_ops);
  46
  47         return nvfs_ops;
  48 }
  49
  50 static inline void nvfs_put_ops(void)
  51 {
  52         percpu_counter_dec(&nvfs_n_ops);
  53 }
  54
  55 static inline bool nvfs_check_feature_set(struct nvfs_dma_rw_ops *ops)
  56 {
  57         bool supported = true;
  58         static time64_t last_printed;
  59
  60         if (unlikely(!NVIDIA_FS_CHECK_FT_SGLIST_PREP(ops))) {
  61                 if ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)
  62                         CDEBUG(D_CONSOLE,
  63                                "NVFS sg list preparation callback missing\n");
  64                 supported = false;
  65         }
  66         if (unlikely(!NVIDIA_FS_CHECK_FT_SGLIST_DMA(ops))) {
  67                 if ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)
  68                         CDEBUG(D_CONSOLE,
  69                                "NVFS DMA mapping callbacks missing\n");
  70                 supported = false;
  71         }
  72         if (unlikely(!NVIDIA_FS_CHECK_FT_GPU_PAGE(ops))) {
  73                 if ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)
  74                         CDEBUG(D_CONSOLE,
  75                                "NVFS page identification callback missing\n");
  76                 supported = false;
  77         }
  78         if (unlikely(!NVIDIA_FS_CHECK_FT_DEVICE_PRIORITY(ops))) {
  79                 if ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)
  80                         CDEBUG(D_CONSOLE,
  81                                "NVFS device priority callback not missing\n");
  82                 supported = false;
  83         }
  84
  85         if (unlikely(!supported &&
  86                      ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)))
  87                 last_printed = ktime_get_seconds();
  88         else if (supported)
  89                 last_printed = 0;
  90
  91         return supported;
  92 }
  93
  94 int REGISTER_FUNC(struct nvfs_dma_rw_ops *ops)
  95 {
  96         if (!ops || !nvfs_check_feature_set(ops))
  97                 return -EINVAL;
  98
  99         nvfs_ops = ops;
 100         (void)percpu_counter_init(&nvfs_n_ops, 0, GFP_KERNEL);
 101         atomic_set(&nvfs_shutdown, 0);
 102         CDEBUG(D_NET, "registering nvfs %p\n", ops);
 103         return 0;
 104 }
 105 EXPORT_SYMBOL(REGISTER_FUNC);
 106
 107 void UNREGISTER_FUNC(void)
 108 {
 109         (void)atomic_cmpxchg(&nvfs_shutdown, 0, 1);
 110         do {
 111                 CDEBUG(D_NET, "Attempting to de-register nvfs: %ld\n",
 112                        nvfs_count_ops());
 113                 msleep(NVFS_HOLD_TIME_MS);
 114         } while (nvfs_count_ops());
 115         nvfs_ops = NULL;
 116         percpu_counter_destroy(&nvfs_n_ops);
 117 }
 118 EXPORT_SYMBOL(UNREGISTER_FUNC);
 119
 120 unsigned int
 121 lnet_get_dev_prio(struct device *dev, unsigned int dev_idx)
 122 {
 123         unsigned int dev_prio = UINT_MAX;
 124         struct nvfs_dma_rw_ops *nvfs_ops;
 125
 126         if (!dev)
 127                 return dev_prio;
 128
 129         nvfs_ops = nvfs_get_ops();
 130         if (!nvfs_ops)
 131                 return dev_prio;
 132
 133         dev_prio = nvfs_ops->nvfs_device_priority (dev, dev_idx);
 134
 135         nvfs_put_ops();
 136         return dev_prio;
 137 }
 138 EXPORT_SYMBOL(lnet_get_dev_prio);
 139
 140 unsigned int
 141 lnet_get_dev_idx(struct page *page)
 142 {
 143         unsigned int dev_idx = UINT_MAX;
 144         struct nvfs_dma_rw_ops *nvfs_ops;
 145
 146         nvfs_ops = nvfs_get_ops();
 147         if (!nvfs_ops)
 148                 return dev_idx;
 149
 150         dev_idx = nvfs_ops->nvfs_gpu_index(page);
 151
 152         nvfs_put_ops();
 153         return dev_idx;
 154 }
 155
 156 int lnet_rdma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
 157                            int nents, enum dma_data_direction direction)
 158 {
 159         struct nvfs_dma_rw_ops *nvfs_ops = nvfs_get_ops();
 160
 161         if (nvfs_ops) {
 162                 int count;
 163
 164                 count = nvfs_ops->nvfs_dma_map_sg_attrs(dev,
 165                                 sg, nents, direction,
 166                                 DMA_ATTR_NO_WARN);
 167
 168                 if (unlikely((count == NVFS_IO_ERR))) {
 169                         nvfs_put_ops();
 170                         return -EIO;
 171                 }
 172
 173                 if (unlikely(count == NVFS_CPU_REQ))
 174                         nvfs_put_ops();
 175                 else
 176                         return count;
 177         }
 178
 179         return 0;
 180 }
 181 EXPORT_SYMBOL(lnet_rdma_map_sg_attrs);
 182
 183 int lnet_rdma_unmap_sg(struct device *dev,
 184                        struct scatterlist *sg, int nents,
 185                        enum dma_data_direction direction)
 186 {
 187         struct nvfs_dma_rw_ops *nvfs_ops = nvfs_get_ops();
 188
 189         if (nvfs_ops) {
 190                 int count;
 191
 192                 count = nvfs_ops->nvfs_dma_unmap_sg(dev, sg,
 193                                                     nents, direction);
 194
 195                 /* drop the count we got by calling nvfs_get_ops() */
 196                 nvfs_put_ops();
 197
 198                 if (count) {
 199                         nvfs_put_ops();
 200                         return count;
 201                 }
 202         }
 203
 204         return 0;
 205 }
 206 EXPORT_SYMBOL(lnet_rdma_unmap_sg);
 207
 208 bool
 209 lnet_is_rdma_only_page(struct page *page)
 210 {
 211         bool is_gpu_page = false;
 212         struct nvfs_dma_rw_ops *nvfs_ops;
 213
 214         LASSERT(page != NULL);
 215
 216         nvfs_ops = nvfs_get_ops();
 217         if (nvfs_ops != NULL) {
 218                 is_gpu_page = nvfs_ops->nvfs_is_gpu_page(page);
 219                 nvfs_put_ops();
 220         }
 221         return is_gpu_page;
 222 }
 223 EXPORT_SYMBOL(lnet_is_rdma_only_page);