LU-8006 ptlrpc: specify ordering of TBF policy rules

With this patch, when inserting a new rule, the rank of the rule
can be given by "start" command. Also, the rank of the rule can be
changed by command of "change".

lctl set_param ost.OSS.ost_io.nrs_tbf_rule=
	"start $NAME jobid={$ID} rate=$RATE rank=$NEXT"
lctl set_param ost.OSS.ost_io.nrs_tbf_rule=
	"change $NAME rate=$RATE rank=$NEXT"

$NAME is the target rule name. $NEXT is the rule name that the target
rule will be moved before.

Signed-off-by: Li Xi <lixi@ddn.com>
Change-Id: I6b465342365d6c09710616cd3c9e068b66a8fc89
Reviewed-on: http://review.whamcloud.com/19476
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Gu Zheng <gzheng@ddn.com>
Reviewed-by: Emoly Liu <emoly.liu@intel.com>
Reviewed-by: Wang Shilong <wshilong@ddn.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/include/lustre_nrs_tbf.h b/lustre/include/lustre_nrs_tbf.h
index 4d93c23..0e5c929 100644 (file)
--- a/lustre/include/lustre_nrs_tbf.h
+++ b/lustre/include/lustre_nrs_tbf.h
@@ -242,9 +242,11 @@ struct nrs_tbf_cmd {
                        char                    *ts_jobids_str;
                        __u32                    ts_valid_type;
                        __u32                    ts_rule_flags;
+                       char                    *ts_next_name;
                } tc_start;
                struct nrs_tbf_cmd_change {
                        __u64                    tc_rpc_rate;
+                       char                    *tc_next_name;
                } tc_change;
        } u;
 };

diff --git a/lustre/ptlrpc/nrs_tbf.c b/lustre/ptlrpc/nrs_tbf.c
index 4d5aa84..4215dcd 100644 (file)
--- a/lustre/ptlrpc/nrs_tbf.c
+++ b/lustre/ptlrpc/nrs_tbf.c
@@ -281,8 +281,11 @@ nrs_tbf_rule_start(struct ptlrpc_nrs_policy *policy,
                   struct nrs_tbf_head *head,
                   struct nrs_tbf_cmd *start)
 {
-       struct nrs_tbf_rule *rule, *tmp_rule;
-       int rc;
+       struct nrs_tbf_rule     *rule;
+       struct nrs_tbf_rule     *tmp_rule;
+       struct nrs_tbf_rule     *next_rule;
+       char                    *next_name = start->u.tc_start.ts_next_name;
+       int                      rc;
 
        rule = nrs_tbf_rule_find(head, start->tc_name);
        if (rule) {
@@ -321,7 +324,21 @@ nrs_tbf_rule_start(struct ptlrpc_nrs_policy *policy,
                nrs_tbf_rule_put(rule);
                return -EEXIST;
        }
-       list_add(&rule->tr_linkage, &head->th_list);
+
+       if (next_name) {
+               next_rule = nrs_tbf_rule_find_nolock(head, next_name);
+               if (!next_rule) {
+                       spin_unlock(&head->th_rule_lock);
+                       nrs_tbf_rule_put(rule);
+                       return -ENOENT;
+               }
+
+               list_add(&rule->tr_linkage, next_rule->tr_linkage.prev);
+               nrs_tbf_rule_put(next_rule);
+       } else {
+               /* Add on the top of the rule list */
+               list_add(&rule->tr_linkage, &head->th_list);
+       }
        spin_unlock(&head->th_rule_lock);
        atomic_inc(&head->th_rule_sequence);
        if (start->u.tc_start.ts_rule_flags & NTRS_DEFAULT) {
@@ -333,20 +350,67 @@ nrs_tbf_rule_start(struct ptlrpc_nrs_policy *policy,
        return 0;
 }
 
+/**
+ * Change the rank of a rule in the rule list
+ *
+ * The matched rule will be moved to the position right before another
+ * given rule.
+ *
+ * \param[in] policy   the policy instance
+ * \param[in] head     the TBF policy instance
+ * \param[in] name     the rule name to be moved
+ * \param[in] next_name        the rule name before which the matched rule will be
+ *                     moved
+ *
+ */
 static int
-nrs_tbf_rule_change(struct ptlrpc_nrs_policy *policy,
-                   struct nrs_tbf_head *head,
-                   struct nrs_tbf_cmd *change)
+nrs_tbf_rule_change_rank(struct ptlrpc_nrs_policy *policy,
+                        struct nrs_tbf_head *head,
+                        char *name,
+                        char *next_name)
+{
+       struct nrs_tbf_rule     *rule = NULL;
+       struct nrs_tbf_rule     *next_rule = NULL;
+       int                      rc = 0;
+
+       LASSERT(head != NULL);
+
+       spin_lock(&head->th_rule_lock);
+       rule = nrs_tbf_rule_find_nolock(head, name);
+       if (!rule)
+               GOTO(out, rc = -ENOENT);
+
+       if (strcmp(name, next_name) == 0)
+               GOTO(out_put, rc);
+
+       next_rule = nrs_tbf_rule_find_nolock(head, next_name);
+       if (!next_rule)
+               GOTO(out_put, rc = -ENOENT);
+
+       list_move(&rule->tr_linkage, next_rule->tr_linkage.prev);
+       nrs_tbf_rule_put(next_rule);
+out_put:
+       nrs_tbf_rule_put(rule);
+out:
+       spin_unlock(&head->th_rule_lock);
+       return rc;
+}
+
+static int
+nrs_tbf_rule_change_rate(struct ptlrpc_nrs_policy *policy,
+                        struct nrs_tbf_head *head,
+                        char *name,
+                        __u64 rate)
 {
        struct nrs_tbf_rule *rule;
 
        assert_spin_locked(&policy->pol_nrs->nrs_lock);
 
-       rule = nrs_tbf_rule_find(head, change->tc_name);
+       rule = nrs_tbf_rule_find(head, name);
        if (rule == NULL)
                return -ENOENT;
 
-       rule->tr_rpc_rate = change->u.tc_change.tc_rpc_rate;
+       rule->tr_rpc_rate = rate;
        rule->tr_nsecs = NSEC_PER_SEC;
        do_div(rule->tr_nsecs, rule->tr_rpc_rate);
        rule->tr_generation++;
@@ -356,6 +420,32 @@ nrs_tbf_rule_change(struct ptlrpc_nrs_policy *policy,
 }
 
 static int
+nrs_tbf_rule_change(struct ptlrpc_nrs_policy *policy,
+                   struct nrs_tbf_head *head,
+                   struct nrs_tbf_cmd *change)
+{
+       __u64    rate = change->u.tc_change.tc_rpc_rate;
+       char    *next_name = change->u.tc_change.tc_next_name;
+       int      rc;
+
+       if (rate != 0) {
+               rc = nrs_tbf_rule_change_rate(policy, head, change->tc_name,
+                                             rate);
+               if (rc)
+                       return rc;
+       }
+
+       if (next_name) {
+               rc = nrs_tbf_rule_change_rank(policy, head, change->tc_name,
+                                             next_name);
+               if (rc)
+                       return rc;
+       }
+
+       return 0;
+}
+
+static int
 nrs_tbf_rule_stop(struct ptlrpc_nrs_policy *policy,
                  struct nrs_tbf_head *head,
                  struct nrs_tbf_cmd *stop)
@@ -1741,6 +1831,16 @@ nrs_tbf_parse_value_pair(struct nrs_tbf_cmd *cmd, char *buffer)
                        cmd->u.tc_change.tc_rpc_rate = rate;
                else
                        return -EINVAL;
+       }  else if (strcmp(key, "rank") == 0) {
+               if (!name_is_valid(val))
+                       return -EINVAL;
+
+               if (cmd->tc_cmd == NRS_CTL_TBF_START_RULE)
+                       cmd->u.tc_start.ts_next_name = val;
+               else if (cmd->tc_cmd == NRS_CTL_TBF_CHANGE_RULE)
+                       cmd->u.tc_change.tc_next_name = val;
+               else
+                       return -EINVAL;
        } else {
                return -EINVAL;
        }
@@ -1768,7 +1868,8 @@ nrs_tbf_parse_value_pairs(struct nrs_tbf_cmd *cmd, char *buffer)
                        cmd->u.tc_start.ts_rpc_rate = tbf_rate;
                break;
        case NRS_CTL_TBF_CHANGE_RULE:
-               if (cmd->u.tc_change.tc_rpc_rate == 0)
+               if (cmd->u.tc_change.tc_rpc_rate == 0 &&
+                   cmd->u.tc_change.tc_next_name == NULL)
                        return -EINVAL;
                break;
        case NRS_CTL_TBF_STOP_RULE:

diff --git a/lustre/tests/sanityn.sh b/lustre/tests/sanityn.sh
index 5f5a3bb..5ecf403 100755 (executable)
--- a/lustre/tests/sanityn.sh
+++ b/lustre/tests/sanityn.sh
@@ -3045,7 +3045,7 @@ tbf_rule_operate()
        do_facet $facet lctl set_param \
                ost.OSS.ost_io.nrs_tbf_rule="$*"
        [ $? -ne 0 ] &&
-               error "failed to operate on TBF rules"
+               error "failed to run operate '$*' on TBF rules"
 }
 
 test_77e() {
@@ -3209,6 +3209,81 @@ test_77h() {
 }
 run_test 77h "Wrong policy name should report error, not LBUG"
 
+tbf_rule_check()
+{
+       local facet=$1
+       local expected=$2
+       local error_message=$3
+       local rule_number=0
+       for rule in $expected; do
+               rule_number=$((rule_number + 1))
+       done
+       local stop_line=$(($rule_number + 3))
+       local awk_command="awk 'NR >= 4 && NR <= $stop_line {print \$1}'"
+
+       local output=$(do_facet $facet lctl get_param \
+               ost.OSS.ost_io.nrs_tbf_rule |
+               eval $awk_command |
+               tr "\n" " " |
+               sed 's/[ ]*$//')
+       if [ "$output" != "$expected" ]; then
+               error "$error_message, expected '$expected', got '$output'"
+       fi
+}
+
+test_77i() {
+    [ $(lustre_version_code ost1) -ge $(version_code 2.8.55) ] ||
+               { skip "Need OST version at least 2.8.55"; return 0; }
+
+       for i in $(seq 1 $OSTCOUNT)
+       do
+               do_facet ost"$i" lctl set_param \
+                       ost.OSS.ost_io.nrs_policies="tbf\ jobid"
+               [ $? -ne 0 ] &&
+                       error "failed to set TBF policy"
+       done
+
+       tbf_rule_check ost1 "default" "error before inserting any rule"
+
+       tbf_rule_operate ost1 "start\ before\ jobid={jobid}\ rate=1000"
+       tbf_rule_check ost1 "before default" \
+               "error when inserting rule 'before'"
+
+       tbf_rule_operate ost1 "start\ after\ jobid={jobid}\ rate=1000\ rank=default"
+       tbf_rule_check ost1 "before after default" \
+               "error when inserting rule 'after'"
+
+       tbf_rule_operate ost1 "start\ target\ jobid={jobid}\ rate=1000\ rank=after"
+       tbf_rule_check ost1 "before target after default" \
+               "error when inserting rule 'target'"
+
+       echo "Move before itself"
+       tbf_rule_operate ost1 "change\ target\ rank=target"
+       tbf_rule_check ost1 "before target after default" \
+               "error when moving before itself"
+
+       echo "Move to higher rank"
+       tbf_rule_operate ost1 "change\ target\ rank=before"
+       tbf_rule_check ost1 "target before after default" \
+               "error when moving to higher rank"
+
+       echo "Move to lower rank"
+       tbf_rule_operate ost1 "change\ target\ rank=after"
+       tbf_rule_check ost1 "before target after default" \
+               "error when moving to lower rank"
+
+       echo "Move before default"
+       tbf_rule_operate ost1 "change\ target\ rank=default"
+       tbf_rule_check ost1 "before after target default" \
+               error "error when moving before default"
+
+       # Cleanup the TBF policy
+       do_nodes $(comma_list $(osts_nodes)) \
+               $LCTL set_param ost.OSS.ost_io.nrs_policies=fifo
+       return 0
+}
+run_test 77i "Change rank of TBF rule"
+
 test_78() { #LU-6673
        local rc