+
+ # sleep 3 seconds to wait the tbf policy stop completely,
+ # or the next test case is possible get -EAGAIN when
+ # setting the tbf policy
+ sleep 3
+}
+run_test 77j "check TBF-OPCode NRS policy"
+
+test_77k() {
+ [[ $(lustre_version_code ost1) -ge $(version_code 2.9.53) ]] ||
+ { skip "Need OST version at least 2.9.53"; return 0; }
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param ost.OSS.ost_io.nrs_policies="tbf" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ext_w\ jobid={dd.$RUNAS_ID}\&opcode={ost_write}\ rate=20" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ext_r\ jobid={dd.$RUNAS_ID}\&opcode={ost_read}\ rate=10"
+
+ nrs_write_read "$RUNAS"
+ tbf_verify 20 10 "$RUNAS"
+
+ local address=$(comma_list "$(host_nids_address $CLIENTS $NETTYPE)")
+ local client_nids=$(nids_list $address "\\")
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param ost.OSS.ost_io.nrs_tbf_rule="stop\ ext_w" \
+ ost.OSS.ost_io.nrs_tbf_rule="stop\ ext_r" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ext_w\ nid={0@lo\ $client_nids}\&opcode={ost_write}\ rate=20" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ext_r\ nid={0@lo\ $client_nids}\&opcode={ost_read}\ rate=10"
+
+ nrs_write_read
+ tbf_verify 20 10
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param ost.OSS.ost_io.nrs_tbf_rule="stop\ ext_w" \
+ ost.OSS.ost_io.nrs_tbf_rule="stop\ ext_r" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ext\ nid={0@lo\ $client_nids}\&jobid={dd.$RUNAS_ID}\ rate=20"
+
+ nrs_write_read "$RUNAS"
+ tbf_verify 20 20 "$RUNAS"
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param ost.OSS.ost_io.nrs_tbf_rule="stop\ ext" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ext_a\ jobid={dd.$RUNAS_ID},opcode={ost_write}\ rate=20" \
+ ost.OSS.ost_io.nrs_tbf_rule="start\ ext_b\ jobid={dd.$RUNAS_ID},opcode={ost_read}\ rate=10"
+
+ nrs_write_read "$RUNAS"
+ # with parameter "RUNAS", it will match the latest rule
+ # "ext_b" first, so the limited write rate is 10.
+ tbf_verify 10 10 "$RUNAS"
+ tbf_verify 20 10
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ lctl set_param ost.OSS.ost_io.nrs_tbf_rule="stop\ ext_a" \
+ ost.OSS.ost_io.nrs_tbf_rule="stop\ ext_b" \
+ ost.OSS.ost_io.nrs_policies="fifo"
+
+ sleep 3
+}
+run_test 77k "check the extended TBF policy with NID/JobID/OPCode expression"
+
+test_77l() {
+ if [ $(lustre_version_code ost1) -lt $(version_code 2.9.54) ]; then
+ skip "Need OST version at least 2.9.54"
+ return 0
+ fi
+
+ local dir=$DIR/$tdir
+
+ mkdir $dir || error "mkdir $dir failed"
+ $LFS setstripe -c $OSTCOUNT $dir || error "setstripe to $dir failed"
+ chmod 777 $dir
+
+ local nodes=$(comma_list $(osts_nodes))
+ do_nodes $nodes lctl set_param ost.OSS.ost_io.nrs_policies=delay \
+ ost.OSS.ost_io.nrs_delay_min=4 \
+ ost.OSS.ost_io.nrs_delay_max=4 \
+ ost.OSS.ost_io.nrs_delay_pct=100
+ [ $? -ne 0 ] && error "Failed to set delay policy"
+
+ local start=$SECONDS
+ do_nodes "${SINGLECLIENT:-$HOSTNAME}" "$RUNAS" \
+ dd if=/dev/zero of="$dir/nrs_delay_$HOSTNAME" bs=1M count=1 \
+ oflag=direct conv=fdatasync ||
+ { do_nodes $nodes lctl set_param ost.OSS.ost_io.nrs_policies="fifo";
+ error "dd on client failed (1)"; }
+ local elapsed=$((SECONDS - start))
+
+ # NRS delay doesn't do sub-second timing, so a request enqueued at
+ # 0.9 seconds can be dequeued at 4.0
+ [ $elapsed -lt 3 ] &&
+ { do_nodes $nodes lctl set_param ost.OSS.ost_io.nrs_policies="fifo";
+ error "Single 1M write should take at least 3 seconds"; }
+
+ start=$SECONDS
+ do_nodes "${SINGLECLIENT:-$HOSTNAME}" "$RUNAS" \
+ dd if=/dev/zero of="$dir/nrs_delay_$HOSTNAME" bs=1M count=10 \
+ oflag=direct conv=fdatasync ||
+ { do_nodes $nodes lctl set_param ost.OSS.ost_io.nrs_policies="fifo";
+ error "dd on client failed (2)"; }
+ elapsed=$((SECONDS - start))
+
+ [ $elapsed -lt 30 ] &&
+ { do_nodes $nodes lctl set_param ost.OSS.ost_io.nrs_policies="fifo";
+ error "Ten 1M writes should take at least 30 seconds"; }
+
+ do_nodes $nodes lctl set_param ost.OSS.ost_io.nrs_policies="fifo"
+ [ $? -ne 0 ] && error "failed to set policy back to fifo"
+