LU-6174 nrs: perform proper division

For NRS TBF both tr_nsecs and tr_rpc_rate are 64 bit values. The
code is wrongly using do_div() which could truncate tr_rpc_rate
to a 32 bit value. With current hardware setups the rpc rate is
less than the 4 billion limit of u32, currently I see 64K of rpcs
with a single IB port. Since this is the case we can change
tr_rpc_rate to a 32 bit value.

Change-Id: I62eabf3a8a30f5c9062aaf4c32fa878710a788c8
Signed-off-by: James Simmons <jsimmons@infradead.org>
Reviewed-on: https://review.whamcloud.com/36749
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Li Xi <lixi@ddn.com>

diff --git a/lustre/include/lustre_nrs_tbf.h b/lustre/include/lustre_nrs_tbf.h
index 0a40719..7141146 100644 (file)
--- a/lustre/include/lustre_nrs_tbf.h
+++ b/lustre/include/lustre_nrs_tbf.h
@@ -102,7 +102,7 @@ struct nrs_tbf_client {
        /** Generation of the rule matched. */
        __u64                            tc_rule_generation;
        /** Limit of RPC rate. */
-       __u64                            tc_rpc_rate;
+       u32                              tc_rpc_rate;
        /** Time to wait for next token. */
        __u64                            tc_nsecs;
        /** RPC token number. */
@@ -169,9 +169,9 @@ struct nrs_tbf_rule {
        /** Generic condition string of the rule. */
        char                            *tr_conds_str;
        /** RPC/s limit. */
-       __u64                            tr_rpc_rate;
+       u32                              tr_rpc_rate;
        /** Time to wait for next token. */
-       __u64                            tr_nsecs;
+       u64                              tr_nsecs_per_rpc;
        /** Token bucket depth. */
        __u64                            tr_depth;
        /** Lock to protect the list of clients. */

diff --git a/lustre/ptlrpc/nrs_tbf.c b/lustre/ptlrpc/nrs_tbf.c
index a8f3b79..31079ae 100644 (file)
--- a/lustre/ptlrpc/nrs_tbf.c
+++ b/lustre/ptlrpc/nrs_tbf.c
@@ -132,7 +132,7 @@ nrs_tbf_cli_reset_value(struct nrs_tbf_head *head,
        struct nrs_tbf_rule *rule = cli->tc_rule;
 
        cli->tc_rpc_rate = rule->tr_rpc_rate;
-       cli->tc_nsecs = rule->tr_nsecs;
+       cli->tc_nsecs = rule->tr_nsecs_per_rpc;
        cli->tc_depth = rule->tr_depth;
        cli->tc_ntoken = rule->tr_depth;
        cli->tc_check_time = ktime_to_ns(ktime_get());
@@ -302,8 +302,7 @@ nrs_tbf_rule_start(struct ptlrpc_nrs_policy *policy,
        memcpy(rule->tr_name, start->tc_name, strlen(start->tc_name));
        rule->tr_rpc_rate = start->u.tc_start.ts_rpc_rate;
        rule->tr_flags = start->u.tc_start.ts_rule_flags;
-       rule->tr_nsecs = NSEC_PER_SEC;
-       do_div(rule->tr_nsecs, rule->tr_rpc_rate);
+       rule->tr_nsecs_per_rpc = NSEC_PER_SEC / rule->tr_rpc_rate;
        rule->tr_depth = tbf_depth;
        atomic_set(&rule->tr_ref, 1);
        INIT_LIST_HEAD(&rule->tr_cli_list);
@@ -350,7 +349,7 @@ nrs_tbf_rule_start(struct ptlrpc_nrs_policy *policy,
                head->th_rule = rule;
        }
 
-       CDEBUG(D_RPCTRACE, "TBF starts rule@%p rate %llu gen %llu\n",
+       CDEBUG(D_RPCTRACE, "TBF starts rule@%p rate %u gen %llu\n",
               rule, rule->tr_rpc_rate, rule->tr_generation);
 
        return 0;
@@ -417,8 +416,7 @@ nrs_tbf_rule_change_rate(struct ptlrpc_nrs_policy *policy,
                return -ENOENT;
 
        rule->tr_rpc_rate = rate;
-       rule->tr_nsecs = NSEC_PER_SEC;
-       do_div(rule->tr_nsecs, rule->tr_rpc_rate);
+       rule->tr_nsecs_per_rpc = NSEC_PER_SEC / rule->tr_rpc_rate;
        rule->tr_generation++;
        nrs_tbf_rule_put(rule);
 
@@ -1002,7 +1000,7 @@ static int nrs_tbf_jobid_rule_init(struct ptlrpc_nrs_policy *policy,
 static int
 nrs_tbf_jobid_rule_dump(struct nrs_tbf_rule *rule, struct seq_file *m)
 {
-       seq_printf(m, "%s {%s} %llu, ref %d\n", rule->tr_name,
+       seq_printf(m, "%s {%s} %u, ref %d\n", rule->tr_name,
                   rule->tr_jobids_str, rule->tr_rpc_rate,
                   atomic_read(&rule->tr_ref) - 1);
        return 0;
@@ -1207,7 +1205,7 @@ static int nrs_tbf_nid_rule_init(struct ptlrpc_nrs_policy *policy,
 static int
 nrs_tbf_nid_rule_dump(struct nrs_tbf_rule *rule, struct seq_file *m)
 {
-       seq_printf(m, "%s {%s} %llu, ref %d\n", rule->tr_name,
+       seq_printf(m, "%s {%s} %u, ref %d\n", rule->tr_name,
                   rule->tr_nids_str, rule->tr_rpc_rate,
                   atomic_read(&rule->tr_ref) - 1);
        return 0;
@@ -2061,7 +2059,7 @@ nrs_tbf_rule_init(struct ptlrpc_nrs_policy *policy,
 static int
 nrs_tbf_generic_rule_dump(struct nrs_tbf_rule *rule, struct seq_file *m)
 {
-       seq_printf(m, "%s %s %llu, ref %d\n", rule->tr_name,
+       seq_printf(m, "%s %s %u, ref %d\n", rule->tr_name,
                   rule->tr_conds_str, rule->tr_rpc_rate,
                   atomic_read(&rule->tr_ref) - 1);
        return 0;
@@ -2355,7 +2353,7 @@ static int nrs_tbf_opcode_rule_init(struct ptlrpc_nrs_policy *policy,
 static int
 nrs_tbf_opcode_rule_dump(struct nrs_tbf_rule *rule, struct seq_file *m)
 {
-       seq_printf(m, "%s {%s} %llu, ref %d\n", rule->tr_name,
+       seq_printf(m, "%s {%s} %u, ref %d\n", rule->tr_name,
                   rule->tr_opcodes_str, rule->tr_rpc_rate,
                   atomic_read(&rule->tr_ref) - 1);
        return 0;
@@ -2670,7 +2668,7 @@ nrs_tbf_id_rule_init(struct ptlrpc_nrs_policy *policy,
 static int
 nrs_tbf_id_rule_dump(struct nrs_tbf_rule *rule, struct seq_file *m)
 {
-       seq_printf(m, "%s {%s} %llu, ref %d\n", rule->tr_name,
+       seq_printf(m, "%s {%s} %u, ref %d\n", rule->tr_name,
                   rule->tr_ids_str, rule->tr_rpc_rate,
                   atomic_read(&rule->tr_ref) - 1);
        return 0;
@@ -2961,7 +2959,7 @@ static int nrs_tbf_res_get(struct ptlrpc_nrs_policy *policy,
                        struct nrs_tbf_rule *rule;
 
                        CDEBUG(D_RPCTRACE,
-                              "TBF class@%p rate %llu sequence %d, "
+                              "TBF class@%p rate %u sequence %d, "
                               "rule flags %d, head sequence %d\n",
                               cli, cli->tc_rpc_rate,
                               cli->tc_rule_sequence,
@@ -3117,8 +3115,8 @@ struct ptlrpc_nrs_request *nrs_tbf_req_get(struct ptlrpc_nrs_policy *policy,
                                                     &cli->tc_node);
                        }
                        CDEBUG(D_RPCTRACE,
-                              "TBF dequeues: class@%p rate %llu gen %llu "
-                              "token %llu, rule@%p rate %llu gen %llu\n",
+                              "TBF dequeues: class@%p rate %u gen %llu "
+                              "token %llu, rule@%p rate %u gen %llu\n",
                               cli, cli->tc_rpc_rate,
                               cli->tc_rule_generation, cli->tc_ntoken,
                               cli->tc_rule, cli->tc_rule->tr_rpc_rate,
@@ -3200,8 +3198,8 @@ static int nrs_tbf_req_add(struct ptlrpc_nrs_policy *policy,
 
        if (rc == 0)
                CDEBUG(D_RPCTRACE,
-                      "TBF enqueues: class@%p rate %llu gen %llu "
-                      "token %llu, rule@%p rate %llu gen %llu\n",
+                      "TBF enqueues: class@%p rate %u gen %llu "
+                      "token %llu, rule@%p rate %u gen %llu\n",
                       cli, cli->tc_rpc_rate,
                       cli->tc_rule_generation, cli->tc_ntoken,
                       cli->tc_rule, cli->tc_rule->tr_rpc_rate,