2 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
4 * Copyright (c) 2011, 2017, Intel Corporation.
6 * Copyright (c) 2018-2020 Data Direct Networks.
8 * This file is part of Lustre, https://wiki.whamcloud.com/
10 * Portals is free software; you can redistribute it and/or
11 * modify it under the terms of version 2 of the GNU General Public
12 * License as published by the Free Software Foundation.
14 * Portals is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
19 * You should have received a copy of the GNU General Public License
20 * version 2 along with this program; If not, see
21 * http://www.gnu.org/licenses/gpl-2.0.html
25 * User Defined Selection Policies (UDSP) are introduced to add
26 * ability of fine traffic control. The policies are instantiated
27 * on LNet constructs and allow preference of some constructs
28 * over others as an extension of the selection algorithm.
29 * The order of operation is defined by the selection algorithm logical flow:
31 * 1. Iterate over all the networks that a peer can be reached on
32 * and select the best local network
33 * - The remote network with the highest priority is examined
35 * - The local network with the highest priority is selected
37 * - The local NI with the highest priority is selected
39 * 2. If the peer is a remote peer and has no local networks,
40 * - then select the remote peer network with the highest priority
42 * - Select the highest priority remote peer_ni on the network selected
44 * - Now that the peer's network and NI are decided, select the router
45 * in round robin from the peer NI's preferred router list.
47 * - Select the highest priority local NI on the local net of the
50 * 3. Otherwise for local peers, select the peer_ni from the peer.
51 * - highest priority peer NI is selected
53 * - Select the peer NI which has the local NI selected on its
57 * Accordingly, the User Interface allows for the following:
58 * - Adding a local network udsp: if multiple local networks are
59 * available, each one can have a priority.
60 * - Adding a local NID udsp: after a local network is chosen,
61 * if there are multiple NIs, each one can have a priority.
62 * - Adding a remote NID udsp: assign priority to a peer NID.
63 * - Adding a NID pair udsp: allows to specify local NIDs
64 * to be added on the list on the specified peer NIs
65 * When selecting a peer NI, the one with the
66 * local NID being used on its list is preferred.
67 * - Adding a Router udsp: similar to the NID pair udsp.
68 * Specified router NIDs are added on the list on the specified peer NIs.
69 * When sending to a remote peer, remote net is selected and the peer NID
70 * is selected. The router which has its nid on the peer NI list
72 * - Deleting a udsp: use the specified policy index to remove it
73 * from the policy list.
75 * Generally, the syntax is as follows
76 * lnetctl udsp add: add a udsp
77 * --src: ip2nets syntax specifying the local NID to match
78 * --dst: ip2nets syntax specifying the remote NID to match
79 * --rte: ip2nets syntax specifying the router NID to match
80 * --priority: priority value (0 - highest priority)
81 * --idx: index of where to insert the rule.
82 * By default, appends to the end of the rule list.
83 * lnetctl udsp del: delete a udsp
84 * --idx: index of the Policy.
85 * lnetctl udsp show: show udsps
86 * --idx: index of the policy to show.
88 * Author: Amir Shehata
91 #include <linux/uaccess.h>
93 #include <lnet/udsp.h>
94 #include <libcfs/libcfs.h>
97 struct lnet_peer_ni *udi_lpni;
98 struct lnet_peer_net *udi_lpn;
99 struct lnet_ni *udi_ni;
100 struct lnet_net *udi_net;
101 struct lnet_ud_nid_descr *udi_match;
102 struct lnet_ud_nid_descr *udi_action;
104 enum lnet_udsp_action_type udi_type;
109 typedef int (*udsp_apply_rule)(struct udsp_info *);
112 UDSP_APPLY_ON_PEERS = 0,
113 UDSP_APPLY_PRIO_ON_NIS = 1,
114 UDSP_APPLY_RTE_ON_NETS = 2,
115 UDSP_APPLY_MAX_ENUM = 3,
118 #define RULE_NOT_APPLICABLE -1
121 lnet_udsp_is_net_rule(struct lnet_ud_nid_descr *match)
123 return list_empty(&match->ud_addr_range);
127 lnet_udsp_expr_list_equal(struct list_head *e1,
128 struct list_head *e2)
130 struct cfs_expr_list *expr1;
131 struct cfs_expr_list *expr2;
132 struct cfs_range_expr *range1, *range2;
134 if (list_empty(e1) && list_empty(e2))
137 if (lnet_get_list_len(e1) != lnet_get_list_len(e2))
140 expr2 = list_first_entry(e2, struct cfs_expr_list, el_link);
142 list_for_each_entry(expr1, e1, el_link) {
143 if (lnet_get_list_len(&expr1->el_exprs) !=
144 lnet_get_list_len(&expr2->el_exprs))
147 range2 = list_first_entry(&expr2->el_exprs,
148 struct cfs_range_expr,
151 list_for_each_entry(range1, &expr1->el_exprs, re_link) {
152 if (range1->re_lo != range2->re_lo ||
153 range1->re_hi != range2->re_hi ||
154 range1->re_stride != range2->re_stride)
156 range2 = list_next_entry(range2, re_link);
158 expr2 = list_next_entry(expr2, el_link);
165 lnet_udsp_nid_descr_equal(struct lnet_ud_nid_descr *e1,
166 struct lnet_ud_nid_descr *e2)
168 if (e1->ud_net_id.udn_net_type != e2->ud_net_id.udn_net_type ||
169 !lnet_udsp_expr_list_equal(&e1->ud_net_id.udn_net_num_range,
170 &e2->ud_net_id.udn_net_num_range) ||
171 !lnet_udsp_expr_list_equal(&e1->ud_addr_range, &e2->ud_addr_range))
178 lnet_udsp_action_equal(struct lnet_udsp *e1, struct lnet_udsp *e2)
180 if (e1->udsp_action_type != e2->udsp_action_type)
183 if (e1->udsp_action_type == EN_LNET_UDSP_ACTION_PRIORITY &&
184 e1->udsp_action.udsp_priority != e2->udsp_action.udsp_priority)
191 lnet_udsp_equal(struct lnet_udsp *e1, struct lnet_udsp *e2)
193 /* check each NID descr */
194 if (!lnet_udsp_nid_descr_equal(&e1->udsp_src, &e2->udsp_src) ||
195 !lnet_udsp_nid_descr_equal(&e1->udsp_dst, &e2->udsp_dst) ||
196 !lnet_udsp_nid_descr_equal(&e1->udsp_rte, &e2->udsp_rte))
202 /* it is enough to look at the net type of the descriptor. If the criteria
203 * is present the net must be specified
206 lnet_udsp_criteria_present(struct lnet_ud_nid_descr *descr)
208 return (descr->ud_net_id.udn_net_type != 0);
212 lnet_udsp_apply_rule_on_ni(struct udsp_info *udi)
215 struct lnet_ni *ni = udi->udi_ni;
216 struct lnet_ud_nid_descr *ni_match = udi->udi_match;
217 __u32 priority = (udi->udi_revert) ? -1 : udi->udi_priority;
219 rc = cfs_match_nid_net(
221 ni_match->ud_net_id.udn_net_type,
222 &ni_match->ud_net_id.udn_net_num_range,
223 &ni_match->ud_addr_range);
227 CDEBUG(D_NET, "apply udsp on ni %s\n",
228 libcfs_nidstr(&ni->ni_nid));
230 /* Detected match. Set NIDs priority */
231 lnet_ni_set_sel_priority_locked(ni, priority);
237 lnet_udsp_apply_rte_list_on_net(struct lnet_net *net,
238 struct lnet_ud_nid_descr *rte_action,
241 struct lnet_remotenet *rnet;
242 struct list_head *rn_list;
243 struct lnet_route *route;
244 struct lnet_peer_ni *lpni;
245 bool cleared = false;
246 struct lnet_nid *gw_nid, *gw_prim_nid;
250 for (i = 0; i < LNET_REMOTE_NETS_HASH_SIZE; i++) {
251 rn_list = &the_lnet.ln_remote_nets_hash[i];
252 list_for_each_entry(rnet, rn_list, lrn_list) {
253 list_for_each_entry(route, &rnet->lrn_routes, lr_list) {
254 /* look if gw nid on the same net matches */
256 &route->lr_gateway->lp_primary_nid;
258 while ((lpni = lnet_get_next_peer_ni_locked(route->lr_gateway,
261 if (!lnet_get_net_locked(lpni->lpni_peer_net->lpn_net_id))
263 gw_nid = &lpni->lpni_nid;
264 rc = cfs_match_nid_net(
266 rte_action->ud_net_id.udn_net_type,
267 &rte_action->ud_net_id.udn_net_num_range,
268 &rte_action->ud_addr_range);
272 /* match gw primary nid on a remote network */
274 gw_nid = gw_prim_nid;
275 rc = cfs_match_nid_net(
277 rte_action->ud_net_id.udn_net_type,
278 &rte_action->ud_net_id.udn_net_num_range,
279 &rte_action->ud_addr_range);
283 lnet_net_unlock(LNET_LOCK_EX);
284 if (!cleared || revert) {
285 lnet_net_clr_pref_rtrs(net);
288 lnet_net_lock(LNET_LOCK_EX);
292 /* match. Add to pref NIDs */
293 CDEBUG(D_NET, "udsp net->gw: %s->%s\n",
294 libcfs_net2str(net->net_id),
295 libcfs_nidstr(gw_prim_nid));
296 rc = lnet_net_add_pref_rtr(net, gw_prim_nid);
297 lnet_net_lock(LNET_LOCK_EX);
298 /* success if EEXIST return */
299 if (rc && rc != -EEXIST) {
300 CERROR("Failed to add %s to %s pref rtr list\n",
301 libcfs_nidstr(gw_prim_nid),
302 libcfs_net2str(net->net_id));
313 lnet_udsp_apply_rte_rule_on_nets(struct udsp_info *udi)
316 int last_failure = 0;
317 struct lnet_net *net;
318 struct lnet_ud_nid_descr *match = udi->udi_match;
319 struct lnet_ud_nid_descr *rte_action = udi->udi_action;
321 list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
322 if (LNET_NETTYP(net->net_id) != match->ud_net_id.udn_net_type)
325 rc = cfs_match_net(net->net_id,
326 match->ud_net_id.udn_net_type,
327 &match->ud_net_id.udn_net_num_range);
331 CDEBUG(D_NET, "apply rule on %s\n",
332 libcfs_net2str(net->net_id));
333 rc = lnet_udsp_apply_rte_list_on_net(net, rte_action,
343 lnet_udsp_apply_rte_rule_on_net(struct udsp_info *udi)
346 struct lnet_net *net = udi->udi_net;
347 struct lnet_ud_nid_descr *match = udi->udi_match;
348 struct lnet_ud_nid_descr *rte_action = udi->udi_action;
350 rc = cfs_match_net(net->net_id,
351 match->ud_net_id.udn_net_type,
352 &match->ud_net_id.udn_net_num_range);
356 CDEBUG(D_NET, "apply rule on %s\n",
357 libcfs_net2str(net->net_id));
358 rc = lnet_udsp_apply_rte_list_on_net(net, rte_action,
365 lnet_udsp_apply_prio_rule_on_net(struct udsp_info *udi)
368 struct lnet_ud_nid_descr *match = udi->udi_match;
369 struct lnet_net *net = udi->udi_net;
370 __u32 priority = (udi->udi_revert) ? -1 : udi->udi_priority;
372 if (!lnet_udsp_is_net_rule(match))
373 return RULE_NOT_APPLICABLE;
375 rc = cfs_match_net(net->net_id,
376 match->ud_net_id.udn_net_type,
377 &match->ud_net_id.udn_net_num_range);
381 CDEBUG(D_NET, "apply rule on %s\n",
382 libcfs_net2str(net->net_id));
384 lnet_net_set_sel_priority_locked(net, priority);
390 lnet_udsp_apply_rule_on_nis(struct udsp_info *udi)
394 struct lnet_net *net;
395 struct lnet_ud_nid_descr *ni_match = udi->udi_match;
396 int last_failure = 0;
398 list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
399 if (LNET_NETTYP(net->net_id) != ni_match->ud_net_id.udn_net_type)
403 if (!lnet_udsp_apply_prio_rule_on_net(udi))
406 list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
408 rc = lnet_udsp_apply_rule_on_ni(udi);
418 lnet_udsp_apply_rte_list_on_lpni(struct lnet_peer_ni *lpni,
419 struct lnet_ud_nid_descr *rte_action,
422 struct lnet_remotenet *rnet;
423 struct list_head *rn_list;
424 struct lnet_route *route;
425 bool cleared = false;
426 struct lnet_nid *gw_nid;
430 for (i = 0; i < LNET_REMOTE_NETS_HASH_SIZE; i++) {
431 rn_list = &the_lnet.ln_remote_nets_hash[i];
432 list_for_each_entry(rnet, rn_list, lrn_list) {
433 list_for_each_entry(route, &rnet->lrn_routes, lr_list) {
434 gw_nid = &route->lr_gateway->lp_primary_nid;
435 rc = cfs_match_nid_net(
437 rte_action->ud_net_id.udn_net_type,
438 &rte_action->ud_net_id.udn_net_num_range,
439 &rte_action->ud_addr_range);
442 lnet_net_unlock(LNET_LOCK_EX);
443 if (!cleared || revert) {
444 CDEBUG(D_NET, "%spref rtr nids from lpni %s\n",
445 (revert) ? "revert " : "clear ",
446 libcfs_nidstr(&lpni->lpni_nid));
447 lnet_peer_clr_pref_rtrs(lpni);
450 lnet_net_lock(LNET_LOCK_EX);
454 CDEBUG(D_NET, "add gw nid %s as preferred for peer %s\n",
455 libcfs_nidstr(gw_nid),
456 libcfs_nidstr(&lpni->lpni_nid));
457 /* match. Add to pref NIDs */
458 rc = lnet_peer_add_pref_rtr(lpni, gw_nid);
459 lnet_net_lock(LNET_LOCK_EX);
460 /* success if EEXIST return */
461 if (rc && rc != -EEXIST) {
462 CERROR("Failed to add %s to %s pref rtr list\n",
463 libcfs_nidstr(gw_nid),
464 libcfs_nidstr(&lpni->lpni_nid));
475 lnet_udsp_apply_ni_list(struct lnet_peer_ni *lpni,
476 struct lnet_ud_nid_descr *ni_action,
481 struct lnet_net *net;
482 bool cleared = false;
484 list_for_each_entry(net, &the_lnet.ln_nets, net_list) {
485 if (LNET_NETTYP(net->net_id) != ni_action->ud_net_id.udn_net_type)
487 list_for_each_entry(ni, &net->net_ni_list, ni_netlist) {
488 rc = cfs_match_nid_net(
490 ni_action->ud_net_id.udn_net_type,
491 &ni_action->ud_net_id.udn_net_num_range,
492 &ni_action->ud_addr_range);
495 lnet_net_unlock(LNET_LOCK_EX);
496 if (!cleared || revert) {
497 lnet_peer_clr_pref_nids(lpni);
498 CDEBUG(D_NET, "%spref nids from lpni %s\n",
499 (revert) ? "revert " : "clear ",
500 libcfs_nidstr(&lpni->lpni_nid));
503 lnet_net_lock(LNET_LOCK_EX);
507 CDEBUG(D_NET, "add nid %s as preferred for peer %s\n",
508 libcfs_nidstr(&ni->ni_nid),
509 libcfs_nidstr(&lpni->lpni_nid));
510 /* match. Add to pref NIDs */
511 rc = lnet_peer_add_pref_nid(lpni, &ni->ni_nid);
512 lnet_net_lock(LNET_LOCK_EX);
513 /* success if EEXIST return */
514 if (rc && rc != -EEXIST) {
515 CERROR("Failed to add %s to %s pref nid list\n",
516 libcfs_nidstr(&ni->ni_nid),
517 libcfs_nidstr(&lpni->lpni_nid));
527 lnet_udsp_apply_rule_on_lpni(struct udsp_info *udi)
530 struct lnet_peer_ni *lpni = udi->udi_lpni;
531 struct lnet_ud_nid_descr *lp_match = udi->udi_match;
532 struct lnet_ud_nid_descr *action = udi->udi_action;
533 __u32 priority = (udi->udi_revert) ? -1 : udi->udi_priority;
534 bool local = udi->udi_local;
535 enum lnet_udsp_action_type type = udi->udi_type;
537 rc = cfs_match_nid_net(
539 lp_match->ud_net_id.udn_net_type,
540 &lp_match->ud_net_id.udn_net_num_range,
541 &lp_match->ud_addr_range);
543 /* check if looking for a net match */
546 lnet_get_list_len(&lp_match->ud_addr_range) ||
547 !cfs_match_net(udi->udi_lpn->lpn_net_id,
548 lp_match->ud_net_id.udn_net_type,
549 &lp_match->ud_net_id.udn_net_num_range))) {
553 if (type == EN_LNET_UDSP_ACTION_PREFERRED_LIST && local) {
554 rc = lnet_udsp_apply_ni_list(lpni, action,
558 } else if (type == EN_LNET_UDSP_ACTION_PREFERRED_LIST &&
560 rc = lnet_udsp_apply_rte_list_on_lpni(lpni, action,
565 lnet_peer_ni_set_selection_priority(lpni, priority);
572 lnet_udsp_apply_rule_on_lpn(struct udsp_info *udi)
575 struct lnet_ud_nid_descr *match = udi->udi_match;
576 struct lnet_peer_net *lpn = udi->udi_lpn;
577 __u32 priority = (udi->udi_revert) ? -1 : udi->udi_priority;
579 if (udi->udi_type == EN_LNET_UDSP_ACTION_PREFERRED_LIST ||
580 !lnet_udsp_is_net_rule(match))
581 return RULE_NOT_APPLICABLE;
583 rc = cfs_match_net(lpn->lpn_net_id,
584 match->ud_net_id.udn_net_type,
585 &match->ud_net_id.udn_net_num_range);
589 CDEBUG(D_NET, "apply rule on lpn %s\n",
590 libcfs_net2str(lpn->lpn_net_id));
591 lnet_peer_net_set_sel_priority_locked(lpn, priority);
597 lnet_udsp_apply_rule_on_lpnis(struct udsp_info *udi)
599 /* iterate over all the peers in the system and find if any of the
600 * peers match the criteria. If they do, clear the preferred list
601 * and add the new list
603 int lncpt = cfs_percpt_number(the_lnet.ln_peer_tables);
604 struct lnet_ud_nid_descr *lp_match = udi->udi_match;
605 struct lnet_peer_table *ptable;
606 struct lnet_peer_net *lpn;
607 struct lnet_peer_ni *lpni;
608 struct lnet_peer *lp;
609 int last_failure = 0;
613 for (cpt = 0; cpt < lncpt; cpt++) {
614 ptable = the_lnet.ln_peer_tables[cpt];
615 list_for_each_entry(lp, &ptable->pt_peer_list, lp_peer_list) {
616 CDEBUG(D_NET, "udsp examining lp %s\n",
617 libcfs_nidstr(&lp->lp_primary_nid));
618 list_for_each_entry(lpn,
621 CDEBUG(D_NET, "udsp examining lpn %s\n",
622 libcfs_net2str(lpn->lpn_net_id));
624 if (LNET_NETTYP(lpn->lpn_net_id) !=
625 lp_match->ud_net_id.udn_net_type)
630 if (!lnet_udsp_apply_rule_on_lpn(udi))
633 list_for_each_entry(lpni,
636 CDEBUG(D_NET, "udsp examining lpni %s\n",
637 libcfs_nidstr(&lpni->lpni_nid));
638 udi->udi_lpni = lpni;
639 rc = lnet_udsp_apply_rule_on_lpni(udi);
651 lnet_udsp_apply_single_policy(struct lnet_udsp *udsp, struct udsp_info *udi,
652 udsp_apply_rule *cbs)
656 if (lnet_udsp_criteria_present(&udsp->udsp_dst) &&
657 lnet_udsp_criteria_present(&udsp->udsp_src)) {
659 if (!cbs[UDSP_APPLY_ON_PEERS])
662 if (udsp->udsp_action_type !=
663 EN_LNET_UDSP_ACTION_PREFERRED_LIST) {
664 CERROR("Bad action type. Expected %d got %d\n",
665 EN_LNET_UDSP_ACTION_PREFERRED_LIST,
666 udsp->udsp_action_type);
669 udi->udi_match = &udsp->udsp_dst;
670 udi->udi_action = &udsp->udsp_src;
671 udi->udi_type = EN_LNET_UDSP_ACTION_PREFERRED_LIST;
672 udi->udi_local = true;
674 CDEBUG(D_NET, "applying udsp (%p) dst->src\n",
676 rc = cbs[UDSP_APPLY_ON_PEERS](udi);
679 } else if (lnet_udsp_criteria_present(&udsp->udsp_dst) &&
680 lnet_udsp_criteria_present(&udsp->udsp_rte)) {
682 if (!cbs[UDSP_APPLY_ON_PEERS])
685 if (udsp->udsp_action_type !=
686 EN_LNET_UDSP_ACTION_PREFERRED_LIST) {
687 CERROR("Bad action type. Expected %d got %d\n",
688 EN_LNET_UDSP_ACTION_PREFERRED_LIST,
689 udsp->udsp_action_type);
693 if (lnet_udsp_criteria_present(&udsp->udsp_src)) {
694 CERROR("only one of src or dst can be specified\n");
697 udi->udi_match = &udsp->udsp_dst;
698 udi->udi_action = &udsp->udsp_rte;
699 udi->udi_type = EN_LNET_UDSP_ACTION_PREFERRED_LIST;
700 udi->udi_local = false;
702 CDEBUG(D_NET, "applying udsp (%p) dst->rte\n",
704 rc = cbs[UDSP_APPLY_ON_PEERS](udi);
707 } else if (lnet_udsp_criteria_present(&udsp->udsp_dst)) {
708 /* destination priority rule */
709 if (!cbs[UDSP_APPLY_ON_PEERS])
712 if (udsp->udsp_action_type !=
713 EN_LNET_UDSP_ACTION_PRIORITY) {
714 CERROR("Bad action type. Expected %d got %d\n",
715 EN_LNET_UDSP_ACTION_PRIORITY,
716 udsp->udsp_action_type);
719 udi->udi_match = &udsp->udsp_dst;
720 udi->udi_type = EN_LNET_UDSP_ACTION_PRIORITY;
721 if (udsp->udsp_action_type !=
722 EN_LNET_UDSP_ACTION_PRIORITY) {
723 udi->udi_priority = 0;
725 udi->udi_priority = udsp->udsp_action.udsp_priority;
727 udi->udi_local = true;
729 CDEBUG(D_NET, "applying udsp (%p) on destination\n",
731 rc = cbs[UDSP_APPLY_ON_PEERS](udi);
734 } else if (lnet_udsp_criteria_present(&udsp->udsp_src)) {
735 /* source priority rule */
736 if (!cbs[UDSP_APPLY_PRIO_ON_NIS])
739 if (udsp->udsp_action_type !=
740 EN_LNET_UDSP_ACTION_PRIORITY) {
741 CERROR("Bad action type. Expected %d got %d\n",
742 EN_LNET_UDSP_ACTION_PRIORITY,
743 udsp->udsp_action_type);
746 udi->udi_match = &udsp->udsp_src;
747 udi->udi_type = EN_LNET_UDSP_ACTION_PRIORITY;
748 if (udsp->udsp_action_type !=
749 EN_LNET_UDSP_ACTION_PRIORITY) {
750 udi->udi_priority = 0;
752 udi->udi_priority = udsp->udsp_action.udsp_priority;
754 udi->udi_local = true;
756 CDEBUG(D_NET, "applying udsp (%p) on source\n",
758 rc = cbs[UDSP_APPLY_PRIO_ON_NIS](udi);
760 CERROR("Bad UDSP policy\n");
768 lnet_udsp_apply_policies_helper(struct lnet_udsp *udsp, struct udsp_info *udi,
769 udsp_apply_rule *cbs)
772 int last_failure = 0;
775 return lnet_udsp_apply_single_policy(udsp, udi, cbs);
777 list_for_each_entry_reverse(udsp,
778 &the_lnet.ln_udsp_list,
780 rc = lnet_udsp_apply_single_policy(udsp, udi, cbs);
789 lnet_udsp_apply_policies_on_ni(struct lnet_ni *ni)
791 struct udsp_info udi;
792 udsp_apply_rule cbs[UDSP_APPLY_MAX_ENUM] = {NULL};
794 memset(&udi, 0, sizeof(udi));
798 cbs[UDSP_APPLY_PRIO_ON_NIS] = lnet_udsp_apply_rule_on_ni;
800 return lnet_udsp_apply_policies_helper(NULL, &udi, cbs);
804 lnet_udsp_apply_policies_on_net(struct lnet_net *net)
806 struct udsp_info udi;
807 udsp_apply_rule cbs[UDSP_APPLY_MAX_ENUM] = {NULL};
809 memset(&udi, 0, sizeof(udi));
813 cbs[UDSP_APPLY_PRIO_ON_NIS] = lnet_udsp_apply_prio_rule_on_net;
814 cbs[UDSP_APPLY_RTE_ON_NETS] = lnet_udsp_apply_rte_rule_on_net;
816 return lnet_udsp_apply_policies_helper(NULL, &udi, cbs);
820 lnet_udsp_apply_policies_on_lpni(struct lnet_peer_ni *lpni)
822 struct udsp_info udi;
823 udsp_apply_rule cbs[UDSP_APPLY_MAX_ENUM] = {NULL};
825 memset(&udi, 0, sizeof(udi));
829 cbs[UDSP_APPLY_ON_PEERS] = lnet_udsp_apply_rule_on_lpni;
831 return lnet_udsp_apply_policies_helper(NULL, &udi, cbs);
835 lnet_udsp_apply_policies_on_lpn(struct lnet_peer_net *lpn)
837 struct udsp_info udi;
838 udsp_apply_rule cbs[UDSP_APPLY_MAX_ENUM] = {NULL};
840 memset(&udi, 0, sizeof(udi));
844 cbs[UDSP_APPLY_ON_PEERS] = lnet_udsp_apply_rule_on_lpn;
846 return lnet_udsp_apply_policies_helper(NULL, &udi, cbs);
850 lnet_udsp_apply_policies(struct lnet_udsp *udsp, bool revert)
853 struct udsp_info udi;
854 udsp_apply_rule cbs[UDSP_APPLY_MAX_ENUM] = {NULL};
856 memset(&udi, 0, sizeof(udi));
858 cbs[UDSP_APPLY_ON_PEERS] = lnet_udsp_apply_rule_on_lpnis;
859 cbs[UDSP_APPLY_PRIO_ON_NIS] = lnet_udsp_apply_rule_on_nis;
860 cbs[UDSP_APPLY_RTE_ON_NETS] = lnet_udsp_apply_rte_rule_on_nets;
862 udi.udi_revert = revert;
864 lnet_net_lock(LNET_LOCK_EX);
865 rc = lnet_udsp_apply_policies_helper(udsp, &udi, cbs);
866 lnet_net_unlock(LNET_LOCK_EX);
872 lnet_udsp_get_policy(int idx)
875 struct lnet_udsp *udsp = NULL;
878 CDEBUG(D_NET, "Get UDSP at idx = %d\n", idx);
883 list_for_each_entry(udsp, &the_lnet.ln_udsp_list, udsp_on_list) {
884 CDEBUG(D_NET, "iterating over upsp %d:%d:%d\n",
885 udsp->udsp_idx, i, idx);
893 CDEBUG(D_NET, "Found UDSP (%p)\n", udsp);
902 lnet_udsp_add_policy(struct lnet_udsp *new, int idx)
904 struct lnet_udsp *udsp;
905 struct lnet_udsp *insert = NULL;
908 list_for_each_entry(udsp, &the_lnet.ln_udsp_list, udsp_on_list) {
909 CDEBUG(D_NET, "found udsp i = %d:%d, idx = %d\n",
910 i, udsp->udsp_idx, idx);
916 if (lnet_udsp_equal(udsp, new)) {
917 if (!lnet_udsp_action_equal(udsp, new) &&
918 udsp->udsp_action_type == EN_LNET_UDSP_ACTION_PRIORITY &&
919 new->udsp_action_type == EN_LNET_UDSP_ACTION_PRIORITY) {
920 udsp->udsp_action.udsp_priority = new->udsp_action.udsp_priority;
921 CDEBUG(D_NET, "udsp: %p index %d updated priority to %d\n",
924 udsp->udsp_action.udsp_priority);
932 list_add(&new->udsp_on_list, insert->udsp_on_list.prev);
934 list_for_each_entry(udsp,
935 &the_lnet.ln_udsp_list,
944 list_add_tail(&new->udsp_on_list, &the_lnet.ln_udsp_list);
948 CDEBUG(D_NET, "udsp: %p added at index %d\n", new, new->udsp_idx);
950 CDEBUG(D_NET, "udsp list:\n");
951 list_for_each_entry(udsp, &the_lnet.ln_udsp_list, udsp_on_list)
952 CDEBUG(D_NET, "udsp %p:%d\n", udsp, udsp->udsp_idx);
958 lnet_udsp_del_policy(int idx)
960 struct lnet_udsp *udsp;
961 struct lnet_udsp *tmp;
962 bool removed = false;
965 lnet_udsp_destroy(false);
969 CDEBUG(D_NET, "del udsp at idx = %d\n", idx);
971 list_for_each_entry_safe(udsp,
973 &the_lnet.ln_udsp_list,
977 if (udsp->udsp_idx == idx && !removed) {
978 list_del_init(&udsp->udsp_on_list);
979 lnet_udsp_apply_policies(udsp, true);
980 lnet_udsp_free(udsp);
992 lnet_udsp_get_ni_info(struct lnet_ioctl_construct_udsp_info *info,
995 struct lnet_nid_list *ne;
996 struct lnet_net *net = ni->ni_net;
1001 info->cud_nid_priority = ni->ni_sel_priority;
1003 info->cud_net_priority = ni->ni_net->net_sel_priority;
1004 list_for_each_entry(ne, &net->net_rtr_pref_nids, nl_list) {
1005 if (i < LNET_MAX_SHOW_NUM_NID)
1006 info->cud_pref_rtr_nid[i] =
1007 lnet_nid_to_nid4(&ne->nl_nid);
1016 lnet_udsp_get_peer_info(struct lnet_ioctl_construct_udsp_info *info,
1017 struct lnet_peer_ni *lpni)
1019 struct lnet_nid_list *ne;
1022 /* peer tree structure needs to be in existence */
1023 LASSERT(lpni && lpni->lpni_peer_net &&
1024 lpni->lpni_peer_net->lpn_peer);
1026 info->cud_nid_priority = lpni->lpni_sel_priority;
1027 CDEBUG(D_NET, "lpni %s has %d pref nids\n",
1028 libcfs_nidstr(&lpni->lpni_nid),
1029 lpni->lpni_pref_nnids);
1030 if (lpni->lpni_pref_nnids == 1) {
1031 info->cud_pref_nid[0] = lnet_nid_to_nid4(&lpni->lpni_pref.nid);
1032 } else if (lpni->lpni_pref_nnids > 1) {
1033 struct list_head *list = &lpni->lpni_pref.nids;
1035 list_for_each_entry(ne, list, nl_list) {
1036 if (i < LNET_MAX_SHOW_NUM_NID)
1037 info->cud_pref_nid[i] =
1038 lnet_nid_to_nid4(&ne->nl_nid);
1046 list_for_each_entry(ne, &lpni->lpni_rtr_pref_nids, nl_list) {
1047 if (i < LNET_MAX_SHOW_NUM_NID)
1048 info->cud_pref_rtr_nid[i] =
1049 lnet_nid_to_nid4(&ne->nl_nid);
1055 info->cud_net_priority = lpni->lpni_peer_net->lpn_sel_priority;
1059 lnet_udsp_get_construct_info(struct lnet_ioctl_construct_udsp_info *info)
1062 struct lnet_peer_ni *lpni;
1063 struct lnet_nid nid;
1066 lnet_nid4_to_nid(info->cud_nid, &nid);
1067 if (!info->cud_peer) {
1068 ni = lnet_nid_to_ni_locked(&nid, 0);
1070 lnet_udsp_get_ni_info(info, ni);
1072 lpni = lnet_peer_ni_find_locked(&nid);
1074 CDEBUG(D_NET, "nid %s is not found\n",
1075 libcfs_nidstr(&nid));
1077 lnet_udsp_get_peer_info(info, lpni);
1078 lnet_peer_ni_decref_locked(lpni);
1085 lnet_udsp_alloc(void)
1087 struct lnet_udsp *udsp;
1089 udsp = kmem_cache_alloc(lnet_udsp_cachep, GFP_NOFS | __GFP_ZERO);
1094 INIT_LIST_HEAD(&udsp->udsp_on_list);
1095 INIT_LIST_HEAD(&udsp->udsp_src.ud_addr_range);
1096 INIT_LIST_HEAD(&udsp->udsp_src.ud_net_id.udn_net_num_range);
1097 INIT_LIST_HEAD(&udsp->udsp_dst.ud_addr_range);
1098 INIT_LIST_HEAD(&udsp->udsp_dst.ud_net_id.udn_net_num_range);
1099 INIT_LIST_HEAD(&udsp->udsp_rte.ud_addr_range);
1100 INIT_LIST_HEAD(&udsp->udsp_rte.ud_net_id.udn_net_num_range);
1102 LIBCFS_ALLOC_POST(udsp, sizeof(*udsp), "alloc");
1107 lnet_udsp_nid_descr_free(struct lnet_ud_nid_descr *nid_descr)
1109 struct list_head *net_range = &nid_descr->ud_net_id.udn_net_num_range;
1111 if (!lnet_udsp_criteria_present(nid_descr))
1114 /* memory management is a bit tricky here. When we allocate the
1115 * memory to store the NID descriptor we allocate a large buffer
1116 * for all the data, so we need to free the entire buffer at
1117 * once. If the net is present the net_range->next points to that
1118 * buffer otherwise if the ud_addr_range is present then it's the
1119 * ud_addr_range.next
1121 if (!list_empty(net_range))
1122 LIBCFS_FREE(net_range->next, nid_descr->ud_mem_size);
1123 else if (!list_empty(&nid_descr->ud_addr_range))
1124 LIBCFS_FREE(nid_descr->ud_addr_range.next,
1125 nid_descr->ud_mem_size);
1129 lnet_udsp_free(struct lnet_udsp *udsp)
1131 lnet_udsp_nid_descr_free(&udsp->udsp_src);
1132 lnet_udsp_nid_descr_free(&udsp->udsp_dst);
1133 lnet_udsp_nid_descr_free(&udsp->udsp_rte);
1135 LIBCFS_FREE_PRE(udsp, sizeof(*udsp), "kfreed");
1136 kmem_cache_free(lnet_udsp_cachep, udsp);
1140 lnet_udsp_destroy(bool shutdown)
1142 struct lnet_udsp *udsp, *tmp;
1144 CDEBUG(D_NET, "Destroying UDSPs in the system\n");
1146 list_for_each_entry_safe(udsp, tmp, &the_lnet.ln_udsp_list,
1148 list_del(&udsp->udsp_on_list);
1150 lnet_udsp_apply_policies(udsp, true);
1151 lnet_udsp_free(udsp);
1156 lnet_size_marshaled_nid_descr(struct lnet_ud_nid_descr *descr)
1158 struct cfs_expr_list *expr;
1160 int range_count = 0;
1161 size_t size = sizeof(struct lnet_ioctl_udsp_descr);
1163 if (!lnet_udsp_criteria_present(descr))
1166 /* we always have one net expression */
1167 if (!list_empty(&descr->ud_net_id.udn_net_num_range)) {
1168 expr = list_first_entry(&descr->ud_net_id.udn_net_num_range,
1169 struct cfs_expr_list, el_link);
1171 /* count the number of cfs_range_expr in the net expression */
1172 range_count = lnet_get_list_len(&expr->el_exprs);
1175 /* count the number of cfs_range_expr in the address expressions */
1176 list_for_each_entry(expr, &descr->ud_addr_range, el_link) {
1178 range_count += lnet_get_list_len(&expr->el_exprs);
1181 size += (sizeof(struct lnet_expressions) * expr_count);
1182 size += (sizeof(struct lnet_range_expr) * range_count);
1188 lnet_get_udsp_size(struct lnet_udsp *udsp)
1190 size_t size = sizeof(struct lnet_ioctl_udsp);
1192 size += lnet_size_marshaled_nid_descr(&udsp->udsp_src);
1193 size += lnet_size_marshaled_nid_descr(&udsp->udsp_dst);
1194 size += lnet_size_marshaled_nid_descr(&udsp->udsp_rte);
1196 CDEBUG(D_NET, "get udsp (%p) size: %d\n", udsp, (int)size);
1202 copy_exprs(struct cfs_expr_list *expr, void __user **bulk,
1205 struct cfs_range_expr *range;
1206 struct lnet_range_expr range_expr;
1208 /* copy over the net range expressions to the bulk */
1209 list_for_each_entry(range, &expr->el_exprs, re_link) {
1210 range_expr.re_lo = range->re_lo;
1211 range_expr.re_hi = range->re_hi;
1212 range_expr.re_stride = range->re_stride;
1213 CDEBUG(D_NET, "Copy Range %u:%u:%u\n",
1214 range_expr.re_lo, range_expr.re_hi,
1215 range_expr.re_stride);
1216 if (copy_to_user(*bulk, &range_expr, sizeof(range_expr))) {
1217 CDEBUG(D_NET, "Failed to copy range_expr\n");
1220 *bulk += sizeof(range_expr);
1221 *bulk_size -= sizeof(range_expr);
1228 copy_nid_range(struct lnet_ud_nid_descr *nid_descr, char *type,
1229 void __user **bulk, __u32 *bulk_size)
1231 struct lnet_ioctl_udsp_descr ioc_udsp_descr;
1232 struct cfs_expr_list *expr;
1233 struct lnet_expressions ioc_expr;
1238 memset(&ioc_udsp_descr, 0, sizeof(ioc_udsp_descr));
1239 ioc_udsp_descr.iud_src_hdr.ud_descr_type = *(__u32 *)type;
1241 /* if criteria not present, copy over the static part of the NID
1244 if (!lnet_udsp_criteria_present(nid_descr)) {
1245 CDEBUG(D_NET, "Descriptor %u:%u:%u:%u\n",
1246 ioc_udsp_descr.iud_src_hdr.ud_descr_type,
1247 ioc_udsp_descr.iud_src_hdr.ud_descr_count,
1248 ioc_udsp_descr.iud_net.ud_net_type,
1249 ioc_udsp_descr.iud_net.ud_net_num_expr.le_count);
1250 if (copy_to_user(*bulk, &ioc_udsp_descr,
1251 sizeof(ioc_udsp_descr))) {
1252 CDEBUG(D_NET, "failed to copy ioc_udsp_descr\n");
1255 *bulk += sizeof(ioc_udsp_descr);
1256 *bulk_size -= sizeof(ioc_udsp_descr);
1260 expr_count = lnet_get_list_len(&nid_descr->ud_addr_range);
1262 /* copy the net information */
1263 if (!list_empty(&nid_descr->ud_net_id.udn_net_num_range)) {
1264 expr = list_first_entry(&nid_descr->ud_net_id.udn_net_num_range,
1265 struct cfs_expr_list, el_link);
1266 net_expr_count = lnet_get_list_len(&expr->el_exprs);
1271 /* set the total expression count */
1272 ioc_udsp_descr.iud_src_hdr.ud_descr_count = expr_count;
1273 ioc_udsp_descr.iud_net.ud_net_type =
1274 nid_descr->ud_net_id.udn_net_type;
1275 ioc_udsp_descr.iud_net.ud_net_num_expr.le_count = net_expr_count;
1277 CDEBUG(D_NET, "Descriptor %u:%u:%u:%u\n",
1278 ioc_udsp_descr.iud_src_hdr.ud_descr_type,
1279 ioc_udsp_descr.iud_src_hdr.ud_descr_count,
1280 ioc_udsp_descr.iud_net.ud_net_type,
1281 ioc_udsp_descr.iud_net.ud_net_num_expr.le_count);
1283 /* copy over the header info to the bulk */
1284 if (copy_to_user(*bulk, &ioc_udsp_descr, sizeof(ioc_udsp_descr))) {
1285 CDEBUG(D_NET, "Failed to copy data\n");
1288 *bulk += sizeof(ioc_udsp_descr);
1289 *bulk_size -= sizeof(ioc_udsp_descr);
1291 /* copy over the net num expression if it exists */
1292 if (net_expr_count) {
1293 rc = copy_exprs(expr, bulk, bulk_size);
1298 /* copy the address range */
1299 list_for_each_entry(expr, &nid_descr->ud_addr_range, el_link) {
1300 ioc_expr.le_count = lnet_get_list_len(&expr->el_exprs);
1301 if (copy_to_user(*bulk, &ioc_expr, sizeof(ioc_expr))) {
1302 CDEBUG(D_NET, "failex to copy ioc_expr\n");
1305 *bulk += sizeof(ioc_expr);
1306 *bulk_size -= sizeof(ioc_expr);
1308 rc = copy_exprs(expr, bulk, bulk_size);
1317 lnet_udsp_marshal(struct lnet_udsp *udsp, struct lnet_ioctl_udsp *ioc_udsp)
1326 bulk = ioc_udsp->iou_bulk;
1327 bulk_size = ioc_udsp->iou_hdr.ioc_len +
1328 ioc_udsp->iou_bulk_size;
1330 CDEBUG(D_NET, "marshal udsp (%p)\n", udsp);
1331 CDEBUG(D_NET, "MEM -----> bulk: %p:0x%x\n", bulk, bulk_size);
1332 /* make sure user space allocated enough buffer to marshal the
1335 if (bulk_size != lnet_get_udsp_size(udsp)) {
1340 ioc_udsp->iou_idx = udsp->udsp_idx;
1341 ioc_udsp->iou_action_type = udsp->udsp_action_type;
1342 ioc_udsp->iou_action.priority = udsp->udsp_action.udsp_priority;
1344 bulk_size -= sizeof(*ioc_udsp);
1346 rc = copy_nid_range(&udsp->udsp_src, "SRC", &bulk, &bulk_size);
1350 rc = copy_nid_range(&udsp->udsp_dst, "DST", &bulk, &bulk_size);
1354 rc = copy_nid_range(&udsp->udsp_rte, "RTE", &bulk, &bulk_size);
1358 CDEBUG(D_NET, "MEM <----- bulk: %p\n", bulk);
1360 /* we should've consumed the entire buffer */
1361 LASSERT(bulk_size == 0);
1365 CERROR("Failed to marshal udsp: %d\n", rc);
1370 copy_range_info(void **bulk, void **buf, struct list_head *list,
1373 struct lnet_range_expr *range_expr;
1374 struct cfs_range_expr *range;
1375 struct cfs_expr_list *exprs;
1376 int range_count = count;
1379 if (range_count == 0)
1382 if (range_count == -1) {
1383 struct lnet_expressions *e;
1386 range_count = e->le_count;
1387 *bulk += sizeof(*e);
1391 INIT_LIST_HEAD(&exprs->el_link);
1392 INIT_LIST_HEAD(&exprs->el_exprs);
1393 list_add_tail(&exprs->el_link, list);
1394 *buf += sizeof(*exprs);
1396 for (i = 0; i < range_count; i++) {
1399 INIT_LIST_HEAD(&range->re_link);
1400 range->re_lo = range_expr->re_lo;
1401 range->re_hi = range_expr->re_hi;
1402 range->re_stride = range_expr->re_stride;
1403 CDEBUG(D_NET, "Copy Range %u:%u:%u\n",
1407 list_add_tail(&range->re_link, &exprs->el_exprs);
1408 *bulk += sizeof(*range_expr);
1409 *buf += sizeof(*range);
1414 copy_ioc_udsp_descr(struct lnet_ud_nid_descr *nid_descr, char *type,
1415 void **bulk, __u32 *bulk_size)
1417 struct lnet_ioctl_udsp_descr *ioc_nid = *bulk;
1418 struct lnet_expressions *exprs;
1421 int range_count = 0;
1424 int remaining_size = *bulk_size;
1428 size_t range_expr_s = sizeof(struct lnet_range_expr);
1429 size_t lnet_exprs_s = sizeof(struct lnet_expressions);
1431 CDEBUG(D_NET, "%s: bulk = %p:%u\n", type, *bulk, *bulk_size);
1433 /* criteria not present, skip over the static part of the
1434 * bulk, which is included for each NID descriptor
1436 if (ioc_nid->iud_net.ud_net_type == 0) {
1437 remaining_size -= sizeof(*ioc_nid);
1438 if (remaining_size < 0) {
1439 CERROR("Truncated userspace udsp buffer given\n");
1442 *bulk += sizeof(*ioc_nid);
1443 *bulk_size = remaining_size;
1447 descr_type = ioc_nid->iud_src_hdr.ud_descr_type;
1448 if (descr_type != *(__u32 *)type) {
1449 CERROR("Bad NID descriptor type. Expected %s, given %c%c%c\n",
1450 type, (__u8)descr_type, (__u8)(descr_type << 4),
1451 (__u8)(descr_type << 8));
1455 /* calculate the total size to verify we have enough buffer.
1456 * Start of by finding how many ranges there are for the net
1459 range_count = ioc_nid->iud_net.ud_net_num_expr.le_count;
1460 size = sizeof(*ioc_nid) + (range_count * range_expr_s);
1461 remaining_size -= size;
1462 if (remaining_size < 0) {
1463 CERROR("Truncated userspace udsp buffer given\n");
1467 CDEBUG(D_NET, "Total net num ranges in %s: %d:%u\n", type,
1469 /* the number of expressions for the NID. IE 4 for IP, 1 for GNI */
1470 expr_count = ioc_nid->iud_src_hdr.ud_descr_count;
1471 CDEBUG(D_NET, "addr as %d exprs\n", expr_count);
1472 /* point tmp to the beginning of the NID expressions */
1474 for (i = 0; i < expr_count; i++) {
1475 /* get the number of ranges per expression */
1477 range_count += exprs->le_count;
1478 size = (range_expr_s * exprs->le_count) + lnet_exprs_s;
1479 remaining_size -= size;
1480 CDEBUG(D_NET, "expr %d:%d:%u:%d:%d\n", i, exprs->le_count,
1481 size, remaining_size, range_count);
1482 if (remaining_size < 0) {
1483 CERROR("Truncated userspace udsp buffer given\n");
1489 *bulk_size = remaining_size;
1491 /* copy over the net type */
1492 nid_descr->ud_net_id.udn_net_type = ioc_nid->iud_net.ud_net_type;
1494 CDEBUG(D_NET, "%u\n", nid_descr->ud_net_id.udn_net_type);
1496 /* allocate the total memory required to copy this NID descriptor */
1497 if (ioc_nid->iud_net.ud_net_num_expr.le_count) {
1498 if (ioc_nid->iud_net.ud_net_num_expr.le_count != 1) {
1499 CERROR("Unexpected number of net numeric ranges \"%u\". Cannot add UDSP rule.\n",
1500 ioc_nid->iud_net.ud_net_num_expr.le_count);
1503 alloc_size = (sizeof(struct cfs_expr_list) * (expr_count + 1)) +
1504 (sizeof(struct cfs_range_expr) * (range_count));
1506 alloc_size = (sizeof(struct cfs_expr_list) * (expr_count)) +
1507 (sizeof(struct cfs_range_expr) * (range_count));
1510 LIBCFS_ALLOC(buf, alloc_size);
1514 /* store the amount of memory allocated so we can free it later on */
1515 nid_descr->ud_mem_size = alloc_size;
1517 /* copy over the net number range */
1518 range_count = ioc_nid->iud_net.ud_net_num_expr.le_count;
1519 *bulk += sizeof(*ioc_nid);
1520 CDEBUG(D_NET, "bulk = %p\n", *bulk);
1521 copy_range_info(bulk, &buf, &nid_descr->ud_net_id.udn_net_num_range,
1523 CDEBUG(D_NET, "bulk = %p\n", *bulk);
1525 /* copy over the NID descriptor */
1526 for (i = 0; i < expr_count; i++) {
1527 copy_range_info(bulk, &buf, &nid_descr->ud_addr_range, -1);
1528 CDEBUG(D_NET, "bulk = %p\n", *bulk);
1535 lnet_udsp_demarshal_add(void *bulk, __u32 bulk_size)
1537 struct lnet_ioctl_udsp *ioc_udsp;
1538 struct lnet_udsp *udsp;
1542 if (bulk_size < sizeof(*ioc_udsp))
1545 udsp = lnet_udsp_alloc();
1551 udsp->udsp_action_type = ioc_udsp->iou_action_type;
1552 udsp->udsp_action.udsp_priority = ioc_udsp->iou_action.priority;
1553 idx = ioc_udsp->iou_idx;
1555 CDEBUG(D_NET, "demarshal descr %u:%u:%d:%u\n", udsp->udsp_action_type,
1556 udsp->udsp_action.udsp_priority, idx, bulk_size);
1558 bulk += sizeof(*ioc_udsp);
1559 bulk_size -= sizeof(*ioc_udsp);
1561 rc = copy_ioc_udsp_descr(&udsp->udsp_src, "SRC", &bulk, &bulk_size);
1565 rc = copy_ioc_udsp_descr(&udsp->udsp_dst, "DST", &bulk, &bulk_size);
1569 rc = copy_ioc_udsp_descr(&udsp->udsp_rte, "RTE", &bulk, &bulk_size);
1573 return lnet_udsp_add_policy(udsp, idx);
1576 lnet_udsp_free(udsp);