4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; If not, see
18 * http://www.gnu.org/licenses/gpl-2.0.html
23 * Copyright 2022 Hewlett Packard Enterprise Development LP
26 * This file is part of Lustre, http://www.lustre.org/
29 * kfilnd device implementation.
31 #include "kfilnd_dev.h"
32 #include "kfilnd_ep.h"
33 #include "kfilnd_dom.h"
34 #include "kfilnd_peer.h"
37 * kfilnd_dev_post_imm_buffers() - Post all immediate receive buffers on each
39 * @dev: KFI LND device to have all endpoint receive buffers posted.
41 * This function should be called only during KFI LND device initialization.
43 * Return: On success, zero. Else, negative errno.
45 int kfilnd_dev_post_imm_buffers(struct kfilnd_dev *dev)
53 for (i = 0; i < dev->kfd_ni->ni_ncpts; i++) {
54 rc = kfilnd_ep_post_imm_buffers(dev->kfd_endpoints[i]);
63 * kfilnd_dev_free() - Free a KFI LND device.
65 * This function will not complete until all underlying KFI LND transactions are
68 * Once the KFI LND device is freed, a reference is returned to the module.
70 void kfilnd_dev_free(struct kfilnd_dev *dev)
78 debugfs_remove_recursive(dev->dev_dir);
80 /* Change state to shutting down so TNs stop using it */
81 dev->kfd_state = KFILND_STATE_SHUTTING_DOWN;
83 /* Cancel all outstanding RX buffers. */
84 for (i = 0; i < dev->kfd_ni->ni_ncpts; i++)
85 kfilnd_ep_cancel_imm_buffers(dev->kfd_endpoints[i]);
87 /* Free all endpoints. */
88 for (i = 0; i < dev->kfd_ni->ni_ncpts; i++)
89 kfilnd_ep_free(dev->kfd_endpoints[i]);
91 kfilnd_peer_destroy(dev);
93 lnet_ncpts = cfs_cpt_number(lnet_cpt_table());
94 LIBCFS_FREE(dev->cpt_to_endpoint,
95 lnet_ncpts * sizeof(*dev->cpt_to_endpoint));
97 LIBCFS_FREE(dev->kfd_endpoints,
98 dev->kfd_ni->ni_ncpts * sizeof(*dev->kfd_endpoints));
100 kfi_close(&dev->kfd_sep->fid);
101 kfi_close(&dev->kfd_av->fid);
103 kfilnd_dom_put(dev->dom);
105 LIBCFS_FREE(dev, sizeof(*dev));
107 module_put(THIS_MODULE);
111 * kfilnd_dev_alloc() - Allocate a new KFI LND device a LNet NI.
112 * @ni: LNet NI used to allocate the KFI LND device.
113 * @node: Node string which can be passed into kfi_getinfo().
115 * During KFI LND device allocation, the LNet NID NID is used to build node
116 * and service string. The LNet NID address (IPv4 address) is used for the node
117 * string. The LNet NID net number is used for the service string. Together, the
118 * node and service string define the address of the KFI LND device.
120 * The node and service strings are used to allocate a KFI scalable endpoint.
121 * The KFI scalable endpoint is later used to allocate KFI LND endpoints.
123 * For each successful KFI LND device allocation, a reference is taken against
124 * this module to it free being prematurely removed.
126 * Return: On success, valid pointer. On error, negative errno pointer.
128 struct kfilnd_dev *kfilnd_dev_alloc(struct lnet_ni *ni,
133 struct kfi_av_attr av_attr = {};
134 struct kfi_info *dev_info;
137 struct kfilnd_dev *dev;
144 /* Start allocating memory and underlying hardware resources for the
147 LIBCFS_ALLOC(dev, sizeof(*dev));
154 spin_lock_init(&dev->kfd_lock);
155 atomic_set(&dev->session_keys, 0);
157 dev->dom = kfilnd_dom_get(ni, node, &dev_info);
158 if (IS_ERR(dev->dom)) {
159 rc = PTR_ERR(dev->dom);
160 CERROR("Failed to get KFI LND domain: rc=%d\n", rc);
164 /* KFI LNet NID address needs to be unique per LNet NID and something
165 * which can be inserted into the KFI AV. The NIC address is one of the
166 * unique components. Local interface NIC address needs to be extracted
167 * and used to build the LNet NID.
169 * At this point, only the KFI CXI provider is supported.
171 if (!dev_info->src_addr ||
172 dev_info->src_addrlen != sizeof(struct kcxi_addr)) {
174 CERROR("No kfabric source address returned\n");
178 dev->nic_addr = ((struct kcxi_addr *)dev_info->src_addr)->nic;
180 /* Create an AV for this device */
181 av_attr.type = KFI_AV_UNSPEC;
182 av_attr.rx_ctx_bits = KFILND_FAB_RX_CTX_BITS;
183 rc = kfi_av_open(dev->dom->domain, &av_attr, &dev->kfd_av, dev);
185 CERROR("Could not open AV, rc = %d\n", rc);
189 /* Create a scalable endpont to represent the device. */
190 rc = kfi_scalable_ep(dev->dom->domain, dev_info, &dev->kfd_sep, dev);
192 CERROR("Could not create scalable endpoint, rc = %d\n", rc);
196 /* Done with info. */
197 kfi_freeinfo(dev_info);
200 /* Bind the endpoint to the AV */
201 rc = kfi_scalable_ep_bind(dev->kfd_sep, &dev->kfd_av->fid, 0);
203 CERROR("Could not bind scalable endpoint to AV, rc = %d\n", rc);
207 /* Enable the scalable endpoint */
208 rc = kfi_enable(dev->kfd_sep);
210 CERROR("Could not enable scalable endpoint, rc = %d\n", rc);
214 /* Allocate an array to store all the KFI LND endpoints. */
215 LIBCFS_ALLOC_GFP(dev->kfd_endpoints,
216 ni->ni_ncpts * sizeof(*dev->kfd_endpoints),
218 if (!dev->kfd_endpoints) {
223 /* Map of all LNet CPTs to endpoints. */
224 lnet_ncpts = cfs_cpt_number(lnet_cpt_table());
225 LIBCFS_ALLOC_GFP(dev->cpt_to_endpoint,
226 lnet_ncpts * sizeof(*dev->cpt_to_endpoint),
228 if (!dev->cpt_to_endpoint) {
230 goto err_free_ep_array;
233 /* Create RX/TX contexts in kfabric for each LNet NI CPT. */
234 for (i = 0; i < ni->ni_ncpts; i++) {
235 cpt = !ni->ni_cpts ? i : ni->ni_cpts[i];
237 dev->kfd_endpoints[i] =
238 kfilnd_ep_alloc(dev, i, cpt,
239 ni->ni_net->net_tunables.lct_max_tx_credits,
240 KFILND_IMMEDIATE_MSG_SIZE);
241 if (IS_ERR(dev->kfd_endpoints[i])) {
242 rc = PTR_ERR(dev->kfd_endpoints[i]);
243 goto err_free_endpoints;
246 dev->cpt_to_endpoint[cpt] = dev->kfd_endpoints[i];
249 kfilnd_peer_init(dev);
251 /* Mark that the dev/NI has now been initialized */
252 dev->kfd_state = KFILND_STATE_INITIALIZED;
254 /* Initialize debugfs stats. */
255 dev->dev_dir = debugfs_create_dir(libcfs_nidstr(&ni->ni_nid),
257 dev->initiator_state_stats_file =
258 debugfs_create_file("initiator_state_stats", 0444,
260 &kfilnd_initiator_state_stats_file_ops);
261 dev->initiator_state_stats_file =
262 debugfs_create_file("initiator_stats", 0444,
264 &kfilnd_initiator_stats_file_ops);
265 dev->initiator_state_stats_file =
266 debugfs_create_file("target_state_stats", 0444, dev->dev_dir,
267 dev, &kfilnd_target_state_stats_file_ops);
268 dev->initiator_state_stats_file =
269 debugfs_create_file("target_stats", 0444, dev->dev_dir, dev,
270 &kfilnd_target_stats_file_ops);
271 dev->initiator_state_stats_file =
272 debugfs_create_file("reset_stats", 0444, dev->dev_dir, dev,
273 &kfilnd_reset_stats_file_ops);
275 kfilnd_dev_reset_stats(dev);
277 try_module_get(THIS_MODULE);
282 for (i = 0; i < ni->ni_ncpts; i++)
283 kfilnd_ep_free(dev->kfd_endpoints[i]);
285 LIBCFS_FREE(dev->cpt_to_endpoint,
286 lnet_ncpts * sizeof(*dev->cpt_to_endpoint));
288 LIBCFS_FREE(dev->kfd_endpoints,
289 ni->ni_ncpts * sizeof(*dev->kfd_endpoints));
291 kfi_close(&dev->kfd_sep->fid);
293 kfi_close(&dev->kfd_av->fid);
295 kfilnd_dom_put(dev->dom);
297 kfi_freeinfo(dev_info);
299 LIBCFS_FREE(dev, sizeof(*dev));
305 void kfilnd_dev_reset_stats(struct kfilnd_dev *dev)
307 unsigned int data_size;
308 enum tn_states state;
309 struct kfilnd_tn_duration_stat *stat;
311 for (data_size = 0; data_size < KFILND_DATA_SIZE_BUCKETS; data_size++) {
312 stat = &dev->initiator_stats.data_size[data_size];
313 atomic64_set(&stat->accumulated_duration, 0);
314 atomic_set(&stat->accumulated_count, 0);
316 stat = &dev->target_stats.data_size[data_size];
317 atomic64_set(&stat->accumulated_duration, 0);
318 atomic_set(&stat->accumulated_count, 0);
320 for (state = 0; state < TN_STATE_MAX; state++) {
321 stat = &dev->initiator_state_stats.state[state].data_size[data_size];
322 atomic64_set(&stat->accumulated_duration, 0);
323 atomic_set(&stat->accumulated_count, 0);
325 stat = &dev->target_state_stats.state[state].data_size[data_size];
326 atomic64_set(&stat->accumulated_duration, 0);
327 atomic_set(&stat->accumulated_count, 0);
332 u32 kfilnd_dev_get_session_key(struct kfilnd_dev *dev)
334 return (u32)atomic_add_return(1, &dev->session_keys);