+test_77j() {
+ local idis
+ local rateis
+ local ost_version=$(lustre_version_code ost1)
+
+ [ $ost_version -ge $(version_code 2.9.53) ] ||
+ { skip "Need OST version at least 2.9.53"; return 0; }
+ if [ $ost_version -ge $(version_code 2.8.60) ]; then
+ idis="opcode="
+ rateis="rate="
+ fi
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param jobid_var=procname_uid \
+ ost.OSS.ost_io.nrs_policies="tbf\ opcode" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ost_r\ ${idis}{ost_read}\ ${rateis}5" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ost_w\ ${idis}{ost_write}\ ${rateis}20"
+ [ $? -ne 0 ] && error "failed to set TBF OPCode policy"
+
+ nrs_write_read
+ tbf_verify 20 5
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param ost.OSS.ost_io.nrs_tbf_rule="stop\ ost_r" \
+ ost.OSS.ost_io.nrs_tbf_rule="stop\ ost_w" \
+ ost.OSS.ost_io.nrs_policies="fifo"
+
+ # sleep 3 seconds to wait the tbf policy stop completely,
+ # or the next test case is possible get -EAGAIN when
+ # setting the tbf policy
+ sleep 3
+}
+run_test 77j "check TBF-OPCode NRS policy"
+
+test_id() {
+ local idstr="${1}id"
+ local policy="${idstr}={$2}"
+ local rate="rate=$3"
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param jobid_var=procname_uid \
+ ost.OSS.ost_io.nrs_policies="tbf\ ${idstr}" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ost_${idstr}\ ${policy}\ ${rate}"
+ [ $? -ne 0 ] && error "failed to set tbf ${idstr} policy"
+
+ nrs_write_read "runas $4"
+ tbf_verify $3 $3 "runas $4"
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param ost.OSS.ost_io.nrs_tbf_rule="stop\ ost_${idstr}" \
+ ost.OSS.ost_io.nrs_policies="fifo"
+
+ # sleep 3 seconds to wait the tbf policy stop completely,
+ # or the next test case is possible get -eagain when
+ # setting the tbf policy
+ sleep 3
+}
+
+test_77ja(){
+ if [ $(lustre_version_code ost1) -lt $(version_code 2.11.50) ]; then
+ skip "Need OST version at least 2.11.50"
+ return 0
+ fi
+ test_id "u" "500" "5" "-u 500"
+ test_id "g" "500" "5" "-u 500 -g 500"
+}
+run_test 77ja "check TBF-UID/GID NRS policy"
+
+cleanup_77k()
+{
+ local rule_lists=$1
+ local old_nrs=$2
+
+ trap 0
+ for rule in $rule_lists; do
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param ost.OSS.ost_io.nrs_tbf_rule="stop\ $rule"
+ done
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param ost.OSS.ost_io.nrs_policies="$old_nrs"
+
+ sleep 3
+}
+
+test_77k() {
+ [[ $(lustre_version_code ost1) -ge $(version_code 2.9.53) ]] ||
+ { skip "Need OST version at least 2.9.53"; return 0; }
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param ost.OSS.ost_io.nrs_policies="tbf" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ext_w\ jobid={dd.$RUNAS_ID}\&opcode={ost_write}\ rate=20" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ext_r\ jobid={dd.$RUNAS_ID}\&opcode={ost_read}\ rate=10"
+
+ nrs_write_read "$RUNAS"
+ tbf_verify 20 10 "$RUNAS"
+
+ local address=$(comma_list "$(host_nids_address $CLIENTS $NETTYPE)")
+ local client_nids=$(nids_list $address "\\")
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param ost.OSS.ost_io.nrs_tbf_rule="stop\ ext_w" \
+ ost.OSS.ost_io.nrs_tbf_rule="stop\ ext_r" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ext_w\ nid={0@lo\ $client_nids}\&opcode={ost_write}\ rate=20" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ext_r\ nid={0@lo\ $client_nids}\&opcode={ost_read}\ rate=10"
+
+ nrs_write_read
+ tbf_verify 20 10
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param ost.OSS.ost_io.nrs_tbf_rule="stop\ ext_w" \
+ ost.OSS.ost_io.nrs_tbf_rule="stop\ ext_r" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ext\ nid={0@lo\ $client_nids}\&jobid={dd.$RUNAS_ID}\ rate=20"
+
+ nrs_write_read "$RUNAS"
+ tbf_verify 20 20 "$RUNAS"
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param ost.OSS.ost_io.nrs_tbf_rule="stop\ ext" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ext_a\ jobid={dd.$RUNAS_ID},opcode={ost_write}\ rate=20" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ext_b\ jobid={dd.$RUNAS_ID},opcode={ost_read}\ rate=10"
+
+ nrs_write_read "$RUNAS"
+ # with parameter "RUNAS", it will match the latest rule
+ # "ext_b" first, so the limited write rate is 10.
+ tbf_verify 10 10 "$RUNAS"
+ tbf_verify 20 10
+
+ trap "cleanup_77k \"ext_a ext_b\" \"fifo\"" EXIT
+
+ [[ $(lustre_version_code ost1) -ge $(version_code 2.10.58) ]] ||
+ { skip "Need OST version at least 2.10.58"; return 0; }
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param ost.OSS.ost_io.nrs_tbf_rule="stop\ ext_a" \
+ ost.OSS.ost_io.nrs_tbf_rule="stop\ ext_b" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ext_ug\ uid={500}\&gid={1000}\ rate=5"
+ nrs_write_read "runas -u 500 -g 1000"
+ tbf_verify 5 5 "runas -u 500 -g 1000"
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param ost.OSS.ost_io.nrs_tbf_rule="stop\ ext_ug" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ext_uw\ uid={500}\&opcode={ost_write}\ rate=20" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ext_ur\ uid={500}\&opcode={ost_read}\ rate=10"
+
+ nrs_write_read "runas -u 500"
+ tbf_verify 20 10 "runas -u 500"
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param ost.OSS.ost_io.nrs_tbf_rule="stop\ ext_uw" \
+ ost.OSS.ost_io.nrs_tbf_rule="stop\ ext_ur" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ext_a\ uid={500},opcode={ost_write}\ rate=20" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ext_b\ uid={500},opcode={ost_read}\ rate=10"
+ nrs_write_read "runas -u 500"
+ tbf_verify 10 10 "runas -u 500"
+ tbf_verify 20 10 "runas -u 500"
+ cleanup_77k "ext_a ext_b" "fifo"
+}
+run_test 77k "check TBF policy with NID/JobID/OPCode expression"
+
+test_77l() {
+ [[ $(lustre_version_code ost1) -ge $(version_code 2.10.56) ]] ||
+ { skip "Need OST version at least 2.10.56"; return 0; }
+
+ do_facet ost1 lctl set_param ost.OSS.ost_io.nrs_policies="tbf\ nid"
+ do_facet ost1 lctl set_param ost.OSS.ost_io.nrs_policies="tbf"
+
+ local output=$(do_facet ost1 lctl get_param \
+ ost.OSS.ost_io.nrs_policies | \
+ awk '/name: tbf/ {print;exit}' | \
+ awk -F ': ' '{print $2}')
+
+ if [ "$output" != "tbf" ]; then
+ error "The generic TBF output is '$output', not 'tbf'"
+ fi
+
+ do_facet ost1 lctl set_param ost.OSS.ost_io.nrs_policies="fifo"
+}
+run_test 77l "check the output of NRS policies for generic TBF"
+
+test_77m() {
+ if [ $(lustre_version_code ost1) -lt $(version_code 2.9.54) ]; then
+ skip "Need OST version at least 2.9.54"
+ return 0
+ fi
+
+ local dir=$DIR/$tdir
+
+ mkdir $dir || error "mkdir $dir failed"
+ $LFS setstripe -c $OSTCOUNT $dir || error "setstripe to $dir failed"
+ chmod 777 $dir
+
+ local nodes=$(comma_list $(osts_nodes))
+ do_nodes $nodes lctl set_param ost.OSS.ost_io.nrs_policies=delay \
+ ost.OSS.ost_io.nrs_delay_min=4 \
+ ost.OSS.ost_io.nrs_delay_max=4 \
+ ost.OSS.ost_io.nrs_delay_pct=100
+ [ $? -ne 0 ] && error "Failed to set delay policy"
+
+ local start=$SECONDS
+ do_nodes "${SINGLECLIENT:-$HOSTNAME}" "$RUNAS" \
+ dd if=/dev/zero of="$dir/nrs_delay_$HOSTNAME" bs=1M count=1 \
+ oflag=direct conv=fdatasync ||
+ { do_nodes $nodes lctl set_param ost.OSS.ost_io.nrs_policies="fifo";
+ error "dd on client failed (1)"; }
+ local elapsed=$((SECONDS - start))
+
+ # NRS delay doesn't do sub-second timing, so a request enqueued at
+ # 0.9 seconds can be dequeued at 4.0
+ [ $elapsed -lt 3 ] &&
+ { do_nodes $nodes lctl set_param ost.OSS.ost_io.nrs_policies="fifo";
+ error "Single 1M write should take at least 3 seconds"; }
+
+ start=$SECONDS
+ do_nodes "${SINGLECLIENT:-$HOSTNAME}" "$RUNAS" \
+ dd if=/dev/zero of="$dir/nrs_delay_$HOSTNAME" bs=1M count=10 \
+ oflag=direct conv=fdatasync ||
+ { do_nodes $nodes lctl set_param ost.OSS.ost_io.nrs_policies="fifo";
+ error "dd on client failed (2)"; }
+ elapsed=$((SECONDS - start))
+
+ [ $elapsed -lt 30 ] &&
+ { do_nodes $nodes lctl set_param ost.OSS.ost_io.nrs_policies="fifo";
+ error "Ten 1M writes should take at least 30 seconds"; }
+
+ do_nodes $nodes lctl set_param ost.OSS.ost_io.nrs_policies="fifo"
+ [ $? -ne 0 ] && error "failed to set policy back to fifo"
+
+ return 0
+}
+run_test 77m "check NRS Delay slows write RPC processing"
+
+test_77n() { #LU-10802
+ if [ $(lustre_version_code ost1) -lt $(version_code 2.10.58) ]; then
+ skip "Need OST version at least 2.10.58"
+ return 0
+ fi
+
+ # Configure jobid_var
+ local saved_jobid_var=$($LCTL get_param -n jobid_var)
+ if [ $saved_jobid_var != procname_uid ]; then
+ set_persistent_param_and_check client \
+ "jobid_var" "$FSNAME.sys.jobid_var" procname_uid
+ fi
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param ost.OSS.ost_io.nrs_policies="tbf\ jobid" \
+ ost.OSS.ost_io.nrs_tbf_rule="stop\ dd_runas" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ dd_runas\ jobid={*.$RUNAS_ID}\ rate=20"
+
+ nrs_write_read
+ tbf_verify 20 20 "$RUNAS"
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param ost.OSS.ost_io.nrs_tbf_rule="stop\ dd_runas" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ dd_runas\ jobid={dd.*}\ rate=20"
+
+ nrs_write_read
+ tbf_verify 20 20
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param ost.OSS.ost_io.nrs_tbf_rule="stop\ dd_runas" \
+ ost.OSS.ost_io.nrs_policies="fifo"
+
+ sleep 3
+
+ local current_jobid_var=$($LCTL get_param -n jobid_var)
+ if [ $saved_jobid_var != $current_jobid_var ]; then
+ set_persistent_param_and_check client \
+ "jobid_var" "$FSNAME.sys.jobid_var" $saved_jobid_var
+ fi
+}
+run_test 77n "check wildcard support for TBF JobID NRS policy"
+