1 #include <lnet/lnet_rdma.h>
2 #include <libcfs/libcfs.h>
3 #include <lnet/lib-lnet.h>
5 #define ERROR_PRINT_DEADLINE 3600
7 atomic_t nvfs_shutdown = ATOMIC_INIT(1);
8 struct nvfs_dma_rw_ops *nvfs_ops = NULL;
9 struct percpu_counter nvfs_n_ops;
11 static inline long nvfs_count_ops(void)
13 return percpu_counter_sum(&nvfs_n_ops);
16 static struct nvfs_dma_rw_ops *nvfs_get_ops(void)
18 if (!nvfs_ops || atomic_read(&nvfs_shutdown))
21 percpu_counter_inc(&nvfs_n_ops);
26 static inline void nvfs_put_ops(void)
28 percpu_counter_dec(&nvfs_n_ops);
31 static inline bool nvfs_check_feature_set(struct nvfs_dma_rw_ops *ops)
33 bool supported = true;
34 static time64_t last_printed;
36 if (unlikely(!NVIDIA_FS_CHECK_FT_SGLIST_PREP(ops))) {
37 if ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)
39 "NVFS sg list preparation callback missing\n");
42 if (unlikely(!NVIDIA_FS_CHECK_FT_SGLIST_DMA(ops))) {
43 if ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)
45 "NVFS DMA mapping callbacks missing\n");
48 if (unlikely(!NVIDIA_FS_CHECK_FT_GPU_PAGE(ops))) {
49 if ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)
51 "NVFS page identification callback missing\n");
54 if (unlikely(!NVIDIA_FS_CHECK_FT_DEVICE_PRIORITY(ops))) {
55 if ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)
57 "NVFS device priority callback not missing\n");
61 if (unlikely(!supported &&
62 ((ktime_get_seconds() - last_printed) > ERROR_PRINT_DEADLINE)))
63 last_printed = ktime_get_seconds();
70 int REGISTER_FUNC(struct nvfs_dma_rw_ops *ops)
72 if (!ops || !nvfs_check_feature_set(ops))
76 (void)percpu_counter_init(&nvfs_n_ops, 0, GFP_KERNEL);
77 atomic_set(&nvfs_shutdown, 0);
78 CDEBUG(D_NET, "registering nvfs %p\n", ops);
81 EXPORT_SYMBOL(REGISTER_FUNC);
83 void UNREGISTER_FUNC(void)
85 (void)atomic_cmpxchg(&nvfs_shutdown, 0, 1);
87 CDEBUG(D_NET, "Attempting to de-register nvfs: %ld\n",
89 msleep(NVFS_HOLD_TIME_MS);
90 } while (nvfs_count_ops());
92 percpu_counter_destroy(&nvfs_n_ops);
94 EXPORT_SYMBOL(UNREGISTER_FUNC);
97 lnet_get_dev_prio(struct device *dev, unsigned int dev_idx)
99 unsigned int dev_prio = UINT_MAX;
100 struct nvfs_dma_rw_ops *nvfs_ops;
105 nvfs_ops = nvfs_get_ops();
109 dev_prio = nvfs_ops->nvfs_device_priority (dev, dev_idx);
114 EXPORT_SYMBOL(lnet_get_dev_prio);
116 int lnet_rdma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
117 int nents, enum dma_data_direction direction)
119 struct nvfs_dma_rw_ops *nvfs_ops = nvfs_get_ops();
124 count = nvfs_ops->nvfs_dma_map_sg_attrs(dev,
125 sg, nents, direction,
128 if (unlikely((count == NVFS_IO_ERR))) {
133 if (unlikely(count == NVFS_CPU_REQ))
141 EXPORT_SYMBOL(lnet_rdma_map_sg_attrs);
143 int lnet_rdma_unmap_sg(struct device *dev,
144 struct scatterlist *sg, int nents,
145 enum dma_data_direction direction)
147 struct nvfs_dma_rw_ops *nvfs_ops = nvfs_get_ops();
152 count = nvfs_ops->nvfs_dma_unmap_sg(dev, sg,
155 /* drop the count we got by calling nvfs_get_ops() */
166 EXPORT_SYMBOL(lnet_rdma_unmap_sg);
169 lnet_is_rdma_only_page(struct page *page)
172 struct nvfs_dma_rw_ops *nvfs_ops;
177 nvfs_ops = nvfs_get_ops();
181 if (!nvfs_ops->nvfs_is_gpu_page(page))
190 EXPORT_SYMBOL(lnet_is_rdma_only_page);
193 lnet_get_dev_idx(struct page *page)
195 unsigned int dev_idx = UINT_MAX;
196 struct nvfs_dma_rw_ops *nvfs_ops;
198 nvfs_ops = nvfs_get_ops();
202 dev_idx = nvfs_ops->nvfs_gpu_index(page);
207 EXPORT_SYMBOL(lnet_get_dev_idx);