From: James Simmons Date: Thu, 5 Mar 2020 16:34:14 +0000 (-0500) Subject: LU-6174 nrs: perform proper division X-Git-Tag: 2.13.53~91 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=c80319213c6dc4ac9826a1bd10c75373e08db837 LU-6174 nrs: perform proper division For NRS TBF both tr_nsecs and tr_rpc_rate are 64 bit values. The code is wrongly using do_div() which could truncate tr_rpc_rate to a 32 bit value. With current hardware setups the rpc rate is less than the 4 billion limit of u32, currently I see 64K of rpcs with a single IB port. Since this is the case we can change tr_rpc_rate to a 32 bit value. Change-Id: I62eabf3a8a30f5c9062aaf4c32fa878710a788c8 Signed-off-by: James Simmons Reviewed-on: https://review.whamcloud.com/36749 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Li Xi --- diff --git a/lustre/include/lustre_nrs_tbf.h b/lustre/include/lustre_nrs_tbf.h index 0a40719..7141146 100644 --- a/lustre/include/lustre_nrs_tbf.h +++ b/lustre/include/lustre_nrs_tbf.h @@ -102,7 +102,7 @@ struct nrs_tbf_client { /** Generation of the rule matched. */ __u64 tc_rule_generation; /** Limit of RPC rate. */ - __u64 tc_rpc_rate; + u32 tc_rpc_rate; /** Time to wait for next token. */ __u64 tc_nsecs; /** RPC token number. */ @@ -169,9 +169,9 @@ struct nrs_tbf_rule { /** Generic condition string of the rule. */ char *tr_conds_str; /** RPC/s limit. */ - __u64 tr_rpc_rate; + u32 tr_rpc_rate; /** Time to wait for next token. */ - __u64 tr_nsecs; + u64 tr_nsecs_per_rpc; /** Token bucket depth. */ __u64 tr_depth; /** Lock to protect the list of clients. */ diff --git a/lustre/ptlrpc/nrs_tbf.c b/lustre/ptlrpc/nrs_tbf.c index a8f3b79..31079ae 100644 --- a/lustre/ptlrpc/nrs_tbf.c +++ b/lustre/ptlrpc/nrs_tbf.c @@ -132,7 +132,7 @@ nrs_tbf_cli_reset_value(struct nrs_tbf_head *head, struct nrs_tbf_rule *rule = cli->tc_rule; cli->tc_rpc_rate = rule->tr_rpc_rate; - cli->tc_nsecs = rule->tr_nsecs; + cli->tc_nsecs = rule->tr_nsecs_per_rpc; cli->tc_depth = rule->tr_depth; cli->tc_ntoken = rule->tr_depth; cli->tc_check_time = ktime_to_ns(ktime_get()); @@ -302,8 +302,7 @@ nrs_tbf_rule_start(struct ptlrpc_nrs_policy *policy, memcpy(rule->tr_name, start->tc_name, strlen(start->tc_name)); rule->tr_rpc_rate = start->u.tc_start.ts_rpc_rate; rule->tr_flags = start->u.tc_start.ts_rule_flags; - rule->tr_nsecs = NSEC_PER_SEC; - do_div(rule->tr_nsecs, rule->tr_rpc_rate); + rule->tr_nsecs_per_rpc = NSEC_PER_SEC / rule->tr_rpc_rate; rule->tr_depth = tbf_depth; atomic_set(&rule->tr_ref, 1); INIT_LIST_HEAD(&rule->tr_cli_list); @@ -350,7 +349,7 @@ nrs_tbf_rule_start(struct ptlrpc_nrs_policy *policy, head->th_rule = rule; } - CDEBUG(D_RPCTRACE, "TBF starts rule@%p rate %llu gen %llu\n", + CDEBUG(D_RPCTRACE, "TBF starts rule@%p rate %u gen %llu\n", rule, rule->tr_rpc_rate, rule->tr_generation); return 0; @@ -417,8 +416,7 @@ nrs_tbf_rule_change_rate(struct ptlrpc_nrs_policy *policy, return -ENOENT; rule->tr_rpc_rate = rate; - rule->tr_nsecs = NSEC_PER_SEC; - do_div(rule->tr_nsecs, rule->tr_rpc_rate); + rule->tr_nsecs_per_rpc = NSEC_PER_SEC / rule->tr_rpc_rate; rule->tr_generation++; nrs_tbf_rule_put(rule); @@ -1002,7 +1000,7 @@ static int nrs_tbf_jobid_rule_init(struct ptlrpc_nrs_policy *policy, static int nrs_tbf_jobid_rule_dump(struct nrs_tbf_rule *rule, struct seq_file *m) { - seq_printf(m, "%s {%s} %llu, ref %d\n", rule->tr_name, + seq_printf(m, "%s {%s} %u, ref %d\n", rule->tr_name, rule->tr_jobids_str, rule->tr_rpc_rate, atomic_read(&rule->tr_ref) - 1); return 0; @@ -1207,7 +1205,7 @@ static int nrs_tbf_nid_rule_init(struct ptlrpc_nrs_policy *policy, static int nrs_tbf_nid_rule_dump(struct nrs_tbf_rule *rule, struct seq_file *m) { - seq_printf(m, "%s {%s} %llu, ref %d\n", rule->tr_name, + seq_printf(m, "%s {%s} %u, ref %d\n", rule->tr_name, rule->tr_nids_str, rule->tr_rpc_rate, atomic_read(&rule->tr_ref) - 1); return 0; @@ -2061,7 +2059,7 @@ nrs_tbf_rule_init(struct ptlrpc_nrs_policy *policy, static int nrs_tbf_generic_rule_dump(struct nrs_tbf_rule *rule, struct seq_file *m) { - seq_printf(m, "%s %s %llu, ref %d\n", rule->tr_name, + seq_printf(m, "%s %s %u, ref %d\n", rule->tr_name, rule->tr_conds_str, rule->tr_rpc_rate, atomic_read(&rule->tr_ref) - 1); return 0; @@ -2355,7 +2353,7 @@ static int nrs_tbf_opcode_rule_init(struct ptlrpc_nrs_policy *policy, static int nrs_tbf_opcode_rule_dump(struct nrs_tbf_rule *rule, struct seq_file *m) { - seq_printf(m, "%s {%s} %llu, ref %d\n", rule->tr_name, + seq_printf(m, "%s {%s} %u, ref %d\n", rule->tr_name, rule->tr_opcodes_str, rule->tr_rpc_rate, atomic_read(&rule->tr_ref) - 1); return 0; @@ -2670,7 +2668,7 @@ nrs_tbf_id_rule_init(struct ptlrpc_nrs_policy *policy, static int nrs_tbf_id_rule_dump(struct nrs_tbf_rule *rule, struct seq_file *m) { - seq_printf(m, "%s {%s} %llu, ref %d\n", rule->tr_name, + seq_printf(m, "%s {%s} %u, ref %d\n", rule->tr_name, rule->tr_ids_str, rule->tr_rpc_rate, atomic_read(&rule->tr_ref) - 1); return 0; @@ -2961,7 +2959,7 @@ static int nrs_tbf_res_get(struct ptlrpc_nrs_policy *policy, struct nrs_tbf_rule *rule; CDEBUG(D_RPCTRACE, - "TBF class@%p rate %llu sequence %d, " + "TBF class@%p rate %u sequence %d, " "rule flags %d, head sequence %d\n", cli, cli->tc_rpc_rate, cli->tc_rule_sequence, @@ -3117,8 +3115,8 @@ struct ptlrpc_nrs_request *nrs_tbf_req_get(struct ptlrpc_nrs_policy *policy, &cli->tc_node); } CDEBUG(D_RPCTRACE, - "TBF dequeues: class@%p rate %llu gen %llu " - "token %llu, rule@%p rate %llu gen %llu\n", + "TBF dequeues: class@%p rate %u gen %llu " + "token %llu, rule@%p rate %u gen %llu\n", cli, cli->tc_rpc_rate, cli->tc_rule_generation, cli->tc_ntoken, cli->tc_rule, cli->tc_rule->tr_rpc_rate, @@ -3200,8 +3198,8 @@ static int nrs_tbf_req_add(struct ptlrpc_nrs_policy *policy, if (rc == 0) CDEBUG(D_RPCTRACE, - "TBF enqueues: class@%p rate %llu gen %llu " - "token %llu, rule@%p rate %llu gen %llu\n", + "TBF enqueues: class@%p rate %u gen %llu " + "token %llu, rule@%p rate %u gen %llu\n", cli, cli->tc_rpc_rate, cli->tc_rule_generation, cli->tc_ntoken, cli->tc_rule, cli->tc_rule->tr_rpc_rate,