From: Li Xi Date: Fri, 15 Jul 2016 01:02:54 +0000 (+0800) Subject: LU-8006 ptlrpc: specify ordering of TBF policy rules X-Git-Tag: 2.8.57~3 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=aa14b0b9a1521b3deb26a07eb8f7bb28ce793001 LU-8006 ptlrpc: specify ordering of TBF policy rules With this patch, when inserting a new rule, the rank of the rule can be given by "start" command. Also, the rank of the rule can be changed by command of "change". lctl set_param ost.OSS.ost_io.nrs_tbf_rule= "start $NAME jobid={$ID} rate=$RATE rank=$NEXT" lctl set_param ost.OSS.ost_io.nrs_tbf_rule= "change $NAME rate=$RATE rank=$NEXT" $NAME is the target rule name. $NEXT is the rule name that the target rule will be moved before. Signed-off-by: Li Xi Change-Id: I6b465342365d6c09710616cd3c9e068b66a8fc89 Reviewed-on: http://review.whamcloud.com/19476 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Gu Zheng Reviewed-by: Emoly Liu Reviewed-by: Wang Shilong Reviewed-by: Oleg Drokin --- diff --git a/lustre/include/lustre_nrs_tbf.h b/lustre/include/lustre_nrs_tbf.h index 4d93c23..0e5c929 100644 --- a/lustre/include/lustre_nrs_tbf.h +++ b/lustre/include/lustre_nrs_tbf.h @@ -242,9 +242,11 @@ struct nrs_tbf_cmd { char *ts_jobids_str; __u32 ts_valid_type; __u32 ts_rule_flags; + char *ts_next_name; } tc_start; struct nrs_tbf_cmd_change { __u64 tc_rpc_rate; + char *tc_next_name; } tc_change; } u; }; diff --git a/lustre/ptlrpc/nrs_tbf.c b/lustre/ptlrpc/nrs_tbf.c index 4d5aa84..4215dcd 100644 --- a/lustre/ptlrpc/nrs_tbf.c +++ b/lustre/ptlrpc/nrs_tbf.c @@ -281,8 +281,11 @@ nrs_tbf_rule_start(struct ptlrpc_nrs_policy *policy, struct nrs_tbf_head *head, struct nrs_tbf_cmd *start) { - struct nrs_tbf_rule *rule, *tmp_rule; - int rc; + struct nrs_tbf_rule *rule; + struct nrs_tbf_rule *tmp_rule; + struct nrs_tbf_rule *next_rule; + char *next_name = start->u.tc_start.ts_next_name; + int rc; rule = nrs_tbf_rule_find(head, start->tc_name); if (rule) { @@ -321,7 +324,21 @@ nrs_tbf_rule_start(struct ptlrpc_nrs_policy *policy, nrs_tbf_rule_put(rule); return -EEXIST; } - list_add(&rule->tr_linkage, &head->th_list); + + if (next_name) { + next_rule = nrs_tbf_rule_find_nolock(head, next_name); + if (!next_rule) { + spin_unlock(&head->th_rule_lock); + nrs_tbf_rule_put(rule); + return -ENOENT; + } + + list_add(&rule->tr_linkage, next_rule->tr_linkage.prev); + nrs_tbf_rule_put(next_rule); + } else { + /* Add on the top of the rule list */ + list_add(&rule->tr_linkage, &head->th_list); + } spin_unlock(&head->th_rule_lock); atomic_inc(&head->th_rule_sequence); if (start->u.tc_start.ts_rule_flags & NTRS_DEFAULT) { @@ -333,20 +350,67 @@ nrs_tbf_rule_start(struct ptlrpc_nrs_policy *policy, return 0; } +/** + * Change the rank of a rule in the rule list + * + * The matched rule will be moved to the position right before another + * given rule. + * + * \param[in] policy the policy instance + * \param[in] head the TBF policy instance + * \param[in] name the rule name to be moved + * \param[in] next_name the rule name before which the matched rule will be + * moved + * + */ static int -nrs_tbf_rule_change(struct ptlrpc_nrs_policy *policy, - struct nrs_tbf_head *head, - struct nrs_tbf_cmd *change) +nrs_tbf_rule_change_rank(struct ptlrpc_nrs_policy *policy, + struct nrs_tbf_head *head, + char *name, + char *next_name) +{ + struct nrs_tbf_rule *rule = NULL; + struct nrs_tbf_rule *next_rule = NULL; + int rc = 0; + + LASSERT(head != NULL); + + spin_lock(&head->th_rule_lock); + rule = nrs_tbf_rule_find_nolock(head, name); + if (!rule) + GOTO(out, rc = -ENOENT); + + if (strcmp(name, next_name) == 0) + GOTO(out_put, rc); + + next_rule = nrs_tbf_rule_find_nolock(head, next_name); + if (!next_rule) + GOTO(out_put, rc = -ENOENT); + + list_move(&rule->tr_linkage, next_rule->tr_linkage.prev); + nrs_tbf_rule_put(next_rule); +out_put: + nrs_tbf_rule_put(rule); +out: + spin_unlock(&head->th_rule_lock); + return rc; +} + +static int +nrs_tbf_rule_change_rate(struct ptlrpc_nrs_policy *policy, + struct nrs_tbf_head *head, + char *name, + __u64 rate) { struct nrs_tbf_rule *rule; assert_spin_locked(&policy->pol_nrs->nrs_lock); - rule = nrs_tbf_rule_find(head, change->tc_name); + rule = nrs_tbf_rule_find(head, name); if (rule == NULL) return -ENOENT; - rule->tr_rpc_rate = change->u.tc_change.tc_rpc_rate; + rule->tr_rpc_rate = rate; rule->tr_nsecs = NSEC_PER_SEC; do_div(rule->tr_nsecs, rule->tr_rpc_rate); rule->tr_generation++; @@ -356,6 +420,32 @@ nrs_tbf_rule_change(struct ptlrpc_nrs_policy *policy, } static int +nrs_tbf_rule_change(struct ptlrpc_nrs_policy *policy, + struct nrs_tbf_head *head, + struct nrs_tbf_cmd *change) +{ + __u64 rate = change->u.tc_change.tc_rpc_rate; + char *next_name = change->u.tc_change.tc_next_name; + int rc; + + if (rate != 0) { + rc = nrs_tbf_rule_change_rate(policy, head, change->tc_name, + rate); + if (rc) + return rc; + } + + if (next_name) { + rc = nrs_tbf_rule_change_rank(policy, head, change->tc_name, + next_name); + if (rc) + return rc; + } + + return 0; +} + +static int nrs_tbf_rule_stop(struct ptlrpc_nrs_policy *policy, struct nrs_tbf_head *head, struct nrs_tbf_cmd *stop) @@ -1741,6 +1831,16 @@ nrs_tbf_parse_value_pair(struct nrs_tbf_cmd *cmd, char *buffer) cmd->u.tc_change.tc_rpc_rate = rate; else return -EINVAL; + } else if (strcmp(key, "rank") == 0) { + if (!name_is_valid(val)) + return -EINVAL; + + if (cmd->tc_cmd == NRS_CTL_TBF_START_RULE) + cmd->u.tc_start.ts_next_name = val; + else if (cmd->tc_cmd == NRS_CTL_TBF_CHANGE_RULE) + cmd->u.tc_change.tc_next_name = val; + else + return -EINVAL; } else { return -EINVAL; } @@ -1768,7 +1868,8 @@ nrs_tbf_parse_value_pairs(struct nrs_tbf_cmd *cmd, char *buffer) cmd->u.tc_start.ts_rpc_rate = tbf_rate; break; case NRS_CTL_TBF_CHANGE_RULE: - if (cmd->u.tc_change.tc_rpc_rate == 0) + if (cmd->u.tc_change.tc_rpc_rate == 0 && + cmd->u.tc_change.tc_next_name == NULL) return -EINVAL; break; case NRS_CTL_TBF_STOP_RULE: diff --git a/lustre/tests/sanityn.sh b/lustre/tests/sanityn.sh index 5f5a3bb..5ecf403 100755 --- a/lustre/tests/sanityn.sh +++ b/lustre/tests/sanityn.sh @@ -3045,7 +3045,7 @@ tbf_rule_operate() do_facet $facet lctl set_param \ ost.OSS.ost_io.nrs_tbf_rule="$*" [ $? -ne 0 ] && - error "failed to operate on TBF rules" + error "failed to run operate '$*' on TBF rules" } test_77e() { @@ -3209,6 +3209,81 @@ test_77h() { } run_test 77h "Wrong policy name should report error, not LBUG" +tbf_rule_check() +{ + local facet=$1 + local expected=$2 + local error_message=$3 + local rule_number=0 + for rule in $expected; do + rule_number=$((rule_number + 1)) + done + local stop_line=$(($rule_number + 3)) + local awk_command="awk 'NR >= 4 && NR <= $stop_line {print \$1}'" + + local output=$(do_facet $facet lctl get_param \ + ost.OSS.ost_io.nrs_tbf_rule | + eval $awk_command | + tr "\n" " " | + sed 's/[ ]*$//') + if [ "$output" != "$expected" ]; then + error "$error_message, expected '$expected', got '$output'" + fi +} + +test_77i() { + [ $(lustre_version_code ost1) -ge $(version_code 2.8.55) ] || + { skip "Need OST version at least 2.8.55"; return 0; } + + for i in $(seq 1 $OSTCOUNT) + do + do_facet ost"$i" lctl set_param \ + ost.OSS.ost_io.nrs_policies="tbf\ jobid" + [ $? -ne 0 ] && + error "failed to set TBF policy" + done + + tbf_rule_check ost1 "default" "error before inserting any rule" + + tbf_rule_operate ost1 "start\ before\ jobid={jobid}\ rate=1000" + tbf_rule_check ost1 "before default" \ + "error when inserting rule 'before'" + + tbf_rule_operate ost1 "start\ after\ jobid={jobid}\ rate=1000\ rank=default" + tbf_rule_check ost1 "before after default" \ + "error when inserting rule 'after'" + + tbf_rule_operate ost1 "start\ target\ jobid={jobid}\ rate=1000\ rank=after" + tbf_rule_check ost1 "before target after default" \ + "error when inserting rule 'target'" + + echo "Move before itself" + tbf_rule_operate ost1 "change\ target\ rank=target" + tbf_rule_check ost1 "before target after default" \ + "error when moving before itself" + + echo "Move to higher rank" + tbf_rule_operate ost1 "change\ target\ rank=before" + tbf_rule_check ost1 "target before after default" \ + "error when moving to higher rank" + + echo "Move to lower rank" + tbf_rule_operate ost1 "change\ target\ rank=after" + tbf_rule_check ost1 "before target after default" \ + "error when moving to lower rank" + + echo "Move before default" + tbf_rule_operate ost1 "change\ target\ rank=default" + tbf_rule_check ost1 "before after target default" \ + error "error when moving before default" + + # Cleanup the TBF policy + do_nodes $(comma_list $(osts_nodes)) \ + $LCTL set_param ost.OSS.ost_io.nrs_policies=fifo + return 0 +} +run_test 77i "Change rank of TBF rule" + test_78() { #LU-6673 local rc