4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; if not, write to the
18 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
19 * Boston, MA 021110-1307, USA
24 * Copyright (c) 2014, Intel Corporation.
27 * This file is part of Lustre, http://www.lustre.org/
28 * Lustre is a trademark of Sun Microsystems, Inc.
30 * lnet/lnet/net_fault.c
32 * Lustre network fault simulation
34 * Author: liang.zhen@intel.com
37 #define DEBUG_SUBSYSTEM S_LNET
39 #include <lnet/lib-lnet.h>
40 #include <lnet/lnetctl.h>
42 #define LNET_MSG_MASK (LNET_PUT_BIT | LNET_ACK_BIT | \
43 LNET_GET_BIT | LNET_REPLY_BIT)
45 struct lnet_drop_rule {
46 /** link chain on the_lnet.ln_drop_rules */
47 struct list_head dr_link;
48 /** attributes of this rule */
49 struct lnet_fault_attr dr_attr;
50 /** lock to protect \a dr_drop_at and \a dr_stat */
53 * the message sequence to drop, which means message is dropped when
54 * dr_stat.drs_count == dr_drop_at
56 unsigned long dr_drop_at;
58 * seconds to drop the next message, it's exclusive with dr_drop_at
60 cfs_time_t dr_drop_time;
61 /** baseline to caculate dr_drop_time */
62 cfs_time_t dr_time_base;
63 /** statistic of dropped messages */
64 struct lnet_fault_stat dr_stat;
68 lnet_fault_nid_match(lnet_nid_t nid, lnet_nid_t msg_nid)
70 if (nid == msg_nid || nid == LNET_NID_ANY)
73 if (LNET_NIDNET(nid) != LNET_NIDNET(msg_nid))
76 /* 255.255.255.255@net is wildcard for all addresses in a network */
77 return LNET_NIDADDR(nid) == LNET_NIDADDR(LNET_NID_ANY);
81 lnet_fault_attr_match(struct lnet_fault_attr *attr, lnet_nid_t src,
82 lnet_nid_t dst, unsigned int type, unsigned int portal)
84 if (!lnet_fault_nid_match(attr->fa_src, src) ||
85 !lnet_fault_nid_match(attr->fa_dst, dst))
88 if (!(attr->fa_msg_mask & (1 << type)))
91 /* NB: ACK and REPLY have no portal, but they should have been
92 * rejected by message mask */
93 if (attr->fa_ptl_mask != 0 && /* has portal filter */
94 !(attr->fa_ptl_mask & (1ULL << portal)))
101 lnet_fault_attr_validate(struct lnet_fault_attr *attr)
103 if (attr->fa_msg_mask == 0)
104 attr->fa_msg_mask = LNET_MSG_MASK; /* all message types */
106 if (attr->fa_ptl_mask == 0) /* no portal filter */
109 /* NB: only PUT and GET can be filtered if portal filter has been set */
110 attr->fa_msg_mask &= LNET_GET_BIT | LNET_PUT_BIT;
111 if (attr->fa_msg_mask == 0) {
112 CDEBUG(D_NET, "can't find valid message type bits %x\n",
120 lnet_fault_stat_inc(struct lnet_fault_stat *stat, unsigned int type)
122 /* NB: fs_counter is NOT updated by this function */
140 * Add a new drop rule to LNet
141 * There is no check for duplicated drop rule, all rules will be checked for
145 lnet_drop_rule_add(struct lnet_fault_attr *attr)
147 struct lnet_drop_rule *rule;
150 if ((attr->u.drop.da_rate == 0) == (attr->u.drop.da_interval == 0)) {
151 CDEBUG(D_NET, "invalid rate %d or interval %d\n",
152 attr->u.drop.da_rate, attr->u.drop.da_interval);
156 if (lnet_fault_attr_validate(attr) != 0)
163 spin_lock_init(&rule->dr_lock);
165 rule->dr_attr = *attr;
166 if (attr->u.drop.da_interval != 0) {
167 rule->dr_time_base = cfs_time_shift(attr->u.drop.da_interval);
168 rule->dr_drop_time = cfs_time_shift(cfs_rand() %
169 attr->u.drop.da_interval);
171 rule->dr_drop_at = cfs_rand() % attr->u.drop.da_rate;
174 lnet_net_lock(LNET_LOCK_EX);
175 list_add(&rule->dr_link, &the_lnet.ln_drop_rules);
176 lnet_net_unlock(LNET_LOCK_EX);
178 CDEBUG(D_NET, "Added drop rule: src %s, dst %s, rate %d, interval %d\n",
179 libcfs_nid2str(attr->fa_src), libcfs_nid2str(attr->fa_src),
180 attr->u.drop.da_rate, attr->u.drop.da_interval);
185 * Remove matched drop rules from lnet, all rules that can match \a src and
186 * \a dst will be removed.
187 * If \a src is zero, then all rules have \a dst as destination will be remove
188 * If \a dst is zero, then all rules have \a src as source will be removed
189 * If both of them are zero, all rules will be removed
192 lnet_drop_rule_del(lnet_nid_t src, lnet_nid_t dst)
194 struct lnet_drop_rule *rule;
195 struct lnet_drop_rule *tmp;
196 struct list_head zombies;
200 INIT_LIST_HEAD(&zombies);
202 lnet_net_lock(LNET_LOCK_EX);
203 list_for_each_entry_safe(rule, tmp, &the_lnet.ln_drop_rules, dr_link) {
204 if (rule->dr_attr.fa_src != src && src != 0)
207 if (rule->dr_attr.fa_dst != dst && dst != 0)
210 list_move(&rule->dr_link, &zombies);
212 lnet_net_unlock(LNET_LOCK_EX);
214 list_for_each_entry_safe(rule, tmp, &zombies, dr_link) {
215 CDEBUG(D_NET, "Remove drop rule: src %s->dst: %s (1/%d, %d)\n",
216 libcfs_nid2str(rule->dr_attr.fa_src),
217 libcfs_nid2str(rule->dr_attr.fa_dst),
218 rule->dr_attr.u.drop.da_rate,
219 rule->dr_attr.u.drop.da_interval);
221 list_del(&rule->dr_link);
230 * List drop rule at position of \a pos
233 lnet_drop_rule_list(int pos, struct lnet_fault_attr *attr,
234 struct lnet_fault_stat *stat)
236 struct lnet_drop_rule *rule;
242 cpt = lnet_net_lock_current();
243 list_for_each_entry(rule, &the_lnet.ln_drop_rules, dr_link) {
247 spin_lock(&rule->dr_lock);
248 *attr = rule->dr_attr;
249 *stat = rule->dr_stat;
250 spin_unlock(&rule->dr_lock);
255 lnet_net_unlock(cpt);
260 * reset counters for all drop rules
263 lnet_drop_rule_reset(void)
265 struct lnet_drop_rule *rule;
269 cpt = lnet_net_lock_current();
271 list_for_each_entry(rule, &the_lnet.ln_drop_rules, dr_link) {
272 struct lnet_fault_attr *attr = &rule->dr_attr;
274 spin_lock(&rule->dr_lock);
276 memset(&rule->dr_stat, 0, sizeof(rule->dr_stat));
277 if (attr->u.drop.da_rate != 0) {
278 rule->dr_drop_at = cfs_rand() % attr->u.drop.da_rate;
280 rule->dr_drop_time = cfs_time_shift(cfs_rand() %
281 attr->u.drop.da_interval);
282 rule->dr_time_base = cfs_time_shift(attr->u.drop.
285 spin_unlock(&rule->dr_lock);
288 lnet_net_unlock(cpt);
293 * check source/destination NID, portal, message type and drop rate,
294 * decide whether should drop this message or not
297 drop_rule_match(struct lnet_drop_rule *rule, lnet_nid_t src,
298 lnet_nid_t dst, unsigned int type, unsigned int portal)
300 struct lnet_fault_attr *attr = &rule->dr_attr;
303 if (!lnet_fault_attr_match(attr, src, dst, type, portal))
306 /* match this rule, check drop rate now */
307 spin_lock(&rule->dr_lock);
308 if (rule->dr_drop_time != 0) { /* time based drop */
309 cfs_time_t now = cfs_time_current();
311 rule->dr_stat.fs_count++;
312 drop = cfs_time_aftereq(now, rule->dr_drop_time);
314 if (cfs_time_after(now, rule->dr_time_base))
315 rule->dr_time_base = now;
317 rule->dr_drop_time = rule->dr_time_base +
318 cfs_time_seconds(cfs_rand() %
319 attr->u.drop.da_interval);
320 rule->dr_time_base += cfs_time_seconds(attr->u.drop.
323 CDEBUG(D_NET, "Drop Rule %s->%s: next drop : "
325 libcfs_nid2str(attr->fa_src),
326 libcfs_nid2str(attr->fa_dst),
330 } else { /* rate based drop */
331 drop = rule->dr_stat.fs_count++ == rule->dr_drop_at;
333 if (rule->dr_stat.fs_count % attr->u.drop.da_rate == 0) {
334 rule->dr_drop_at = rule->dr_stat.fs_count +
335 cfs_rand() % attr->u.drop.da_rate;
336 CDEBUG(D_NET, "Drop Rule %s->%s: next drop: %lu\n",
337 libcfs_nid2str(attr->fa_src),
338 libcfs_nid2str(attr->fa_dst), rule->dr_drop_at);
342 if (drop) { /* drop this message, update counters */
343 lnet_fault_stat_inc(&rule->dr_stat, type);
344 rule->dr_stat.u.drop.ds_dropped++;
347 spin_unlock(&rule->dr_lock);
352 * Check if message from \a src to \a dst can match any existed drop rule
355 lnet_drop_rule_match(lnet_hdr_t *hdr)
357 struct lnet_drop_rule *rule;
358 lnet_nid_t src = le64_to_cpu(hdr->src_nid);
359 lnet_nid_t dst = le64_to_cpu(hdr->dest_nid);
360 unsigned int typ = le32_to_cpu(hdr->type);
361 unsigned int ptl = -1;
365 /* NB: if Portal is specified, then only PUT and GET will be
366 * filtered by drop rule */
367 if (typ == LNET_MSG_PUT)
368 ptl = le32_to_cpu(hdr->msg.put.ptl_index);
369 else if (typ == LNET_MSG_GET)
370 ptl = le32_to_cpu(hdr->msg.get.ptl_index);
372 cpt = lnet_net_lock_current();
373 list_for_each_entry(rule, &the_lnet.ln_drop_rules, dr_link) {
374 drop = drop_rule_match(rule, src, dst, typ, ptl);
379 lnet_net_unlock(cpt);
384 lnet_fault_ctl(int opc, struct libcfs_ioctl_data *data)
386 struct lnet_fault_attr *attr;
387 struct lnet_fault_stat *stat;
389 attr = (struct lnet_fault_attr *)data->ioc_inlbuf1;
395 case LNET_CTL_DROP_ADD:
399 return lnet_drop_rule_add(attr);
401 case LNET_CTL_DROP_DEL:
405 data->ioc_count = lnet_drop_rule_del(attr->fa_src,
409 case LNET_CTL_DROP_RESET:
410 lnet_drop_rule_reset();
413 case LNET_CTL_DROP_LIST:
414 stat = (struct lnet_fault_stat *)data->ioc_inlbuf2;
415 if (attr == NULL || stat == NULL)
418 return lnet_drop_rule_list(data->ioc_count, attr, stat);
423 lnet_fault_init(void)
425 CLASSERT(LNET_PUT_BIT == 1 << LNET_MSG_PUT);
426 CLASSERT(LNET_ACK_BIT == 1 << LNET_MSG_ACK);
427 CLASSERT(LNET_GET_BIT == 1 << LNET_MSG_GET);
428 CLASSERT(LNET_REPLY_BIT == 1 << LNET_MSG_REPLY);
434 lnet_fault_fini(void)
436 lnet_drop_rule_del(0, 0);
438 LASSERT(list_empty(&the_lnet.ln_drop_rules));