+test_70a() {
+ local test_dir=$tdir/test_dir
+
+ mkdir -p $DIR1/$tdir
+ if [ $MDSCOUNT -ge 2 ]; then
+ local MDTIDX=1
+ $LFS mkdir -i $MDTIDX $DIR1/$test_dir ||
+ error "Create remote directory failed"
+ else
+ mkdir -p $DIR1/$test_dir
+ fi
+ cd $DIR2/$test_dir || error "cd directory failed"
+ rm -rf $DIR1/$test_dir || error "unlink directory failed"
+
+ cd $DIR2/$tdir || error "exit directory"
+}
+run_test 70a "cd directory && rm directory"
+
+test_70b() { # LU-2781
+ local i
+ mkdir -p $DIR1/$tdir
+
+ touch $DIR1/$tdir/file
+ for ((i = 0; i < 32; i++)); do
+ $LFS rm_entry $DIR1/$tdir/non_existent_dir &>/dev/null
+ done
+ rm $DIR1/$tdir/file || error "cannot remove file after rm_entry"
+
+ touch $DIR1/$tdir/file
+ $LFS mkdir -i0 $DIR1/$tdir/test_dir
+ $LFS rm_entry $DIR1/$tdir/test_dir &>/dev/null
+ rm -rf $DIR1/$tdir/test_dir ||
+ error "cannot remove directory after rm_entry"
+ rm $DIR1/$tdir/file || error "cannot remove file after rm_entry"
+}
+run_test 70b "remove files after calling rm_entry"
+
+test_71() {
+ local server_version=$(lustre_version_code $SINGLEMDS)
+
+ [[ $server_version -lt $(version_code 2.1.6) ]] &&
+ skip "Need MDS version at least 2.1.6" && return
+
+ # Patch not applied to 2.2 and 2.3 branches
+ [[ $server_version -ge $(version_code 2.2.0) ]] &&
+ [[ $server_version -lt $(version_code 2.4.0) ]] &&
+ skip "Need MDS version at least 2.4.0" && return
+
+ checkfiemap --test ||
+ { skip "checkfiemap not runnable: $?" && return; }
+ # write data this way: hole - data - hole - data
+ dd if=/dev/urandom of=$DIR1/$tfile bs=40K seek=1 count=1
+ [ "$(facet_fstype ost$(($($GETSTRIPE -i $DIR1/$tfile) + 1)))" = \
+ "zfs" ] &&
+ skip "ORI-366/LU-1941: FIEMAP unimplemented on ZFS" && return 0
+ dd if=/dev/urandom of=$DIR1/$tfile bs=40K seek=3 count=1
+ GET_STAT="lctl get_param -n ldlm.services.ldlm_cbd.stats"
+ stat $DIR2/$tfile
+ local can1=$($GET_STAT | awk '/ldlm_bl_callback/ {print $2}')
+ echo $can1
+ checkfiemap $DIR2/$tfile 81920 ||
+ error "data is not flushed from client"
+ local can2=$($GET_STAT | awk '/ldlm_bl_callback/ {print $2}')
+ echo $can2
+
+ # common case of "create file, copy file" on a single node
+ # should not flush data from ost
+ dd if=/dev/urandom of=$DIR1/$tfile bs=40K seek=1 count=1
+ dd if=/dev/urandom of=$DIR1/$tfile bs=40K seek=3 count=1
+ stat $DIR1/$tfile
+ local can3=$($GET_STAT | awk '/ldlm_bl_callback/ {print $2}')
+ echo $can3
+ checkfiemap $DIR1/$tfile 81920 ||
+ error 4
+ local can4=$($GET_STAT | awk '/ldlm_bl_callback/ {print $2}')
+ echo $can2
+ [ $can3 -eq $can4 ] || error $((can2-can1)) "cancel RPC occured."
+}
+run_test 71 "correct file map just after write operation is finished"
+
+test_72() {
+ local p="$TMP/sanityN-$TESTNAME.parameters"
+ local tlink1
+ local tlink2
+ save_lustre_params client "llite.*.xattr_cache" > $p
+ lctl set_param llite.*.xattr_cache 1 ||
+ { skip "xattr cache is not supported"; return 0; }
+
+ touch $DIR1/$tfile
+ setfattr -n user.attr1 -v value1 $DIR1/$tfile ||
+ error "setfattr1 failed"
+ getfattr -n user.attr1 $DIR2/$tfile | grep value1 ||
+ error "getfattr1 failed"
+ setfattr -n user.attr1 -v value2 $DIR2/$tfile ||
+ error "setfattr2 failed"
+ getfattr -n user.attr1 $DIR1/$tfile | grep value2 ||
+ error "getfattr2 failed"
+
+ # check that trusted.link is consistent
+ tlink1=$(getfattr -n trusted.link $DIR1/$tfile | md5sum)
+ ln $DIR2/$tfile $DIR2/$tfile-2 || error "failed to link"
+ tlink2=$(getfattr -n trusted.link $DIR1/$tfile | md5sum)
+ echo "$tlink1 $tlink2"
+ [ "$tlink1" = "$tlink2" ] && error "trusted.link should have changed!"
+
+ rm -f $DIR2/$tfile
+
+ restore_lustre_params < $p
+ rm -f $p
+}
+run_test 72 "getxattr/setxattr cache should be consistent between nodes"
+
+test_73() {
+ local p="$TMP/sanityN-$TESTNAME.parameters"
+ save_lustre_params client "llite.*.xattr_cache" > $p
+ lctl set_param llite.*.xattr_cache 1 ||
+ { skip "xattr cache is not supported"; return 0; }
+
+ touch $DIR1/$tfile
+ setfattr -n user.attr1 -v value1 $DIR1/$tfile ||
+ error "setfattr1 failed"
+ getfattr -n user.attr1 $DIR2/$tfile || error "getfattr1 failed"
+ getfattr -n user.attr1 $DIR1/$tfile || error "getfattr2 failed"
+ clear_llite_stats
+ # PR lock should be cached by now on both clients
+ getfattr -n user.attr1 $DIR1/$tfile || error "getfattr3 failed"
+ # 2 hits for getfattr(0)+getfattr(size)
+ [ $(calc_llite_stats getxattr_hits) -eq 2 ] || error "not cached in $DIR1"
+ getfattr -n user.attr1 $DIR2/$tfile || error "getfattr4 failed"
+ # 4 hits for more getfattr(0)+getfattr(size)
+ [ $(calc_llite_stats getxattr_hits) -eq 4 ] || error "not cached in $DIR2"
+ rm -f $DIR2/$tfile
+
+ restore_lustre_params < $p
+ rm -f $p
+}
+run_test 73 "getxattr should not cause xattr lock cancellation"
+
+test_74() {
+ [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.4.93) ] &&
+ skip "Need MDS version at least 2.4.93" && return
+
+ dd if=/dev/zero of=$DIR1/$tfile-1 bs=1K count=1
+ dd if=/dev/zero of=$DIR1/$tfile-2 bs=1K count=1
+ flocks_test 4 $DIR1/$tfile-1 $DIR2/$tfile-2
+}
+run_test 74 "flock deadlock: different mounts =============="
+
+# LU-3889
+test_75() {
+ $LFS setstripe -c 2 -S 1m -i 0 $DIR1/$tfile
+ dd if=/dev/zero of=$DIR1/$tfile bs=1M count=2
+ cancel_lru_locks osc
+
+ dd of=$DIR1/$tfile if=/dev/zero bs=1M count=1 seek=1 conv=notrunc
+ sync
+
+ # define OBD_FAIL_LDLM_ENQUEUE_HANG 0x31d
+ $LCTL set_param fail_loc=0x31d
+ stat -c %s $DIR1/$tfile &
+ local pid=$!
+ sleep 1
+ kill -9 $pid
+
+ # For bad lock error handler we should ASSERT and got kernel panic here
+ sleep 4
+ $LCTL set_param fail_loc=0
+}
+run_test 75 "osc: upcall after unuse lock==================="
+
+test_76() { #LU-946
+ [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.5.53) ]] &&
+ skip "Need MDS version at least 2.5.53" && return
+
+ remote_mds_nodsh && skip "remote MDS with nodsh" && return
+ local fcount=2048
+ declare -a fd_list
+ declare -a fid_list
+
+ if remote_mds; then
+ nid=$($LCTL list_nids | sed "s/\./\\\./g")
+ else
+ nid="0@lo"
+ fi
+
+ rm -rf $DIR/$tdir
+ test_mkdir -p $DIR/$tdir
+
+ # drop all open locks and close any cached "open" files on the client
+ cancel_lru_locks mdc
+
+ echo -n "open files "
+ ulimit -n 8096
+ for ((i = 0; i < $fcount; i++)); do
+ touch $DIR/$tdir/f_$i
+ local fd=$(free_fd)
+ local cmd="exec $fd<$DIR/$tdir/f_$i"
+ eval $cmd
+ fd_list[i]=$fd
+ echo -n "."
+ done
+ echo
+
+ local get_open_fids="$LCTL get_param -n mdt.*.exports.'$nid'.open_files"
+ local fid_list=($(do_nodes $(comma_list $(mdts_nodes)) $get_open_fids))
+
+ # Possible errors in openfiles FID list.
+ # 1. Missing FIDs. Check 1
+ # 2. Extra FIDs. Check 1
+ # 3. Duplicated FID. Check 2
+ # 4. Invalid FIDs. Check 2
+ # 5. Valid FID, points to some other file. Check 3
+
+ # Check 1
+ [ ${#fid_list[@]} -ne $fcount ] &&
+ error "${#fid_list[@]} != $fcount open files"
+
+ for (( i = 0; i < $fcount; i++ )) ; do
+ cmd="exec ${fd_list[i]}</dev/null"
+ eval $cmd
+ filename=$($LFS fid2path $DIR2 ${fid_list[i]})
+
+ # Check 2
+ rm --interactive=no $filename
+ [ $? -ne 0 ] &&
+ error "Nonexisting fid ${fid_list[i]} listed."
+ done
+
+ # Check 3
+ ls_op=$(ls $DIR2/$tdir | wc -l)
+ [ $ls_op -ne 0 ] &&
+ error "Some openfiles are missing in lproc output"
+
+ rm -rf $DIR/$tdir
+}
+run_test 76 "Verify open file for 2048 files"
+
+nrs_write_read() {
+ local n=16
+ local dir=$DIR/$tdir
+ local myRUNAS="$1"
+
+ mkdir $dir || error "mkdir $dir failed"
+ $LFS setstripe -c $OSTCOUNT $dir || error "setstripe to $dir failed"
+ chmod 777 $dir
+
+ do_nodes $CLIENTS $myRUNAS dd if=/dev/zero of="$dir/nrs_r_$HOSTNAME"\
+ bs=1M count=$n > /dev/null 2>&1
+
+ for ((i = 0; i < $n; i++)); do
+ do_nodes $CLIENTS $myRUNAS dd if=/dev/zero\
+ of="$dir/nrs_w_$HOSTNAME" bs=1M seek=$i count=1\
+ > /dev/null 2>&1 &
+ local pids_w[$i]=$!
+ done
+ do_nodes $CLIENTS sync;
+ cancel_lru_locks osc
+
+ for ((i = 0; i < $n; i++)); do
+ do_nodes $CLIENTS $myRUNAS dd if="$dir/nrs_w_$HOSTNAME"\
+ of=/dev/zero bs=1M seek=$i count=1 > /dev/null 2>&1 &
+ local pids_r[$i]=$!
+ done
+ cancel_lru_locks osc
+
+ for ((i = 0; i < $n; i++)); do
+ wait ${pids_w[$i]}
+ wait ${pids_r[$i]}
+ done
+ rm -rf $dir || error "rm -rf $dir failed"
+}
+
+test_77a() { #LU-3266
+ do_facet $SINGLEMDS lctl set_param ost.OSS.*.nrs_policies="fifo"
+ nrs_write_read
+
+ return 0
+}
+run_test 77a "check FIFO NRS policy"
+
+
+test_77b() { #LU-3266
+ do_facet $SINGLEMDS lctl set_param ost.OSS.*.nrs_policies="crrn"
+ do_facet $SINGLEMDS lctl set_param ost.OSS.*.nrs_crrn_quantum=1
+
+ echo "policy: crr-n, crrn_quantum 1"
+ nrs_write_read
+
+ do_facet $SINGLEMDS lctl set_param ost.OSS.*.nrs_crrn_quantum=64
+
+ echo "policy: crr-n, crrn_quantum 64"
+ nrs_write_read
+
+ return 0
+}
+run_test 77b "check CRR-N NRS policy"
+
+orr_trr() {
+ local policy=$1
+
+ for i in $(seq 1 $OSTCOUNT)
+ do
+ do_facet ost"$i" lctl set_param \
+ ost.OSS.ost_io.nrs_policies=$policy
+ do_facet ost"$i" lctl set_param \
+ ost.OSS.*.nrs_"$policy"_quantum=1
+ do_facet ost"$i" lctl set_param \
+ ost.OSS.*.nrs_"$policy"_offset_type="physical"
+ do_facet ost"$i" lctl set_param \
+ ost.OSS.*.nrs_"$policy"_supported="reads"
+ done
+
+ echo "policy: $policy, ${policy}_quantum 1, ${policy}_offset_type \
+ physical, ${policy}_supported reads"
+ nrs_write_read
+
+ for i in $(seq 1 $OSTCOUNT)
+ do
+ do_facet ost"$i" lctl set_param \
+ ost.OSS.*.nrs_${policy}_supported="writes"
+ do_facet ost"$i" lctl set_param \
+ ost.OSS.*.nrs_${policy}_quantum=64
+ done
+ echo "policy: $policy, ${policy}_quantum 64, \
+ ${policy}_offset_type physical, ${policy}_supported writes"
+ nrs_write_read
+
+ for i in $(seq 1 $OSTCOUNT)
+ do
+ do_facet ost"$i" lctl set_param \
+ ost.OSS.*.nrs_${policy}_supported="reads_and_writes"
+ do_facet ost"$i" lctl set_param \
+ ost.OSS.*.nrs_${policy}_offset_type="logical"
+ done
+ echo "policy: $policy, ${policy}_quantum 64, \
+ ${policy}_offset_type logical, ${policy}_supported reads_and_writes"
+ nrs_write_read
+
+ return 0
+}
+
+test_77c() { #LU-3266
+ orr_trr "orr"
+ return 0
+}
+run_test 77c "check ORR NRS policy"
+
+test_77d() { #LU-3266
+ orr_trr "trr"
+ return 0
+}
+run_test 77d "check TRR nrs policy"
+
+tbf_rule_operate()
+{
+ local facet=$1
+ shift 1
+
+ do_facet $facet lctl set_param \
+ ost.OSS.ost_io.nrs_tbf_rule="$@"
+ [ $? -ne 0 ] &&
+ error "failed to operate on TBF rules"
+}
+
+test_77e() {
+ for i in $(seq 1 $OSTCOUNT)
+ do
+ do_facet ost"$i" lctl set_param \
+ ost.OSS.ost_io.nrs_policies="tbf\ nid"
+ [ $? -ne 0 ] &&
+ error "failed to set TBF policy"
+ done
+
+ # Only operate rules on ost0 since OSTs might run on the same OSS
+ # Add some rules
+ tbf_rule_operate ost0 "start\ localhost\ {0@lo}\ 1000"
+ local address=$(comma_list "$(host_nids_address $CLIENTS $NETTYPE)")
+ local client_nids=$(nids_list $address "\\")
+ tbf_rule_operate ost0 "start\ clients\ {$client_nids}\ 100"
+ tbf_rule_operate ost0 "start\ others\ {*.*.*.*@$NETTYPE}\ 50"
+ nrs_write_read
+
+ # Change the rules
+ tbf_rule_operate ost0 "change\ localhost\ 1001"
+ tbf_rule_operate ost0 "change\ clients\ 101"
+ tbf_rule_operate ost0 "change\ others\ 51"
+ nrs_write_read
+
+ # Stop the rules
+ tbf_rule_operate ost0 "stop\ localhost"
+ tbf_rule_operate ost0 "stop\ clients"
+ tbf_rule_operate ost0 "stop\ others"
+ nrs_write_read
+
+ # Cleanup the TBF policy
+ for i in $(seq 1 $OSTCOUNT)
+ do
+ do_facet ost"$i" lctl set_param \
+ ost.OSS.ost_io.nrs_policies="fifo"
+ [ $? -ne 0 ] &&
+ error "failed to set policy back to fifo"
+ done
+ nrs_write_read
+ return 0
+}
+run_test 77e "check TBF NID nrs policy"
+
+test_77f() {
+ # Configure jobid_var
+ local saved_jobid_var=$($LCTL get_param -n jobid_var)
+ if [ $saved_jobid_var != procname_uid ]; then
+ set_conf_param_and_check client \
+ "$LCTL get_param -n jobid_var" \
+ "$FSNAME.sys.jobid_var" procname_uid
+ fi
+
+ for i in $(seq 1 $OSTCOUNT)
+ do
+ do_facet ost"$i" lctl set_param \
+ ost.OSS.ost_io.nrs_policies="tbf\ jobid"
+ [ $? -ne 0 ] &&
+ error "failed to set TBF policy"
+ done
+
+ # Only operate rules on ost0 since OSTs might run on the same OSS
+ # Add some rules
+ tbf_rule_operate ost0 "start\ runas\ {iozone.$RUNAS_ID\ dd.$RUNAS_ID\ tiotest.$RUNAS_ID}\ 1000"
+ tbf_rule_operate ost0 "start\ iozone_runas\ {iozone.$RUNAS_ID}\ 100"
+ tbf_rule_operate ost0 "start\ dd_runas\ {dd.$RUNAS_ID}\ 50"
+ nrs_write_read "$RUNAS"
+
+ # Change the rules
+ tbf_rule_operate ost0 "change\ runas\ 1001"
+ tbf_rule_operate ost0 "change\ iozone_runas\ 101"
+ tbf_rule_operate ost0 "change\ dd_runas\ 51"
+ nrs_write_read "$RUNAS"
+
+ # Stop the rules
+ tbf_rule_operate ost0 "stop\ runas"
+ tbf_rule_operate ost0 "stop\ iozone_runas"
+ tbf_rule_operate ost0 "stop\ dd_runas"
+ nrs_write_read "$RUNAS"
+
+ # Cleanup the TBF policy
+ for i in $(seq 1 $OSTCOUNT)
+ do
+ do_facet ost"$i" lctl set_param \
+ ost.OSS.ost_io.nrs_policies="fifo"
+ [ $? -ne 0 ] &&
+ error "failed to set policy back to fifo"
+ done
+ nrs_write_read "$RUNAS"
+
+ local current_jobid_var=$($LCTL get_param -n jobid_var)
+ if [ $saved_jobid_var != $current_jobid_var ]; then
+ set_conf_param_and_check client \
+ "$LCTL get_param -n jobid_var" \
+ "$FSNAME.sys.jobid_var" $saved_jobid_var
+ fi
+ return 0
+}
+run_test 77f "check TBF JobID nrs policy"
+
+test_77g() {
+ for i in $(seq 1 $OSTCOUNT)
+ do
+ do_facet ost"$i" lctl set_param \
+ ost.OSS.ost_io.nrs_policies="tbf\ nid"
+ [ $? -ne 0 ] &&
+ error "failed to set TBF policy"
+ done
+
+ for i in $(seq 1 $OSTCOUNT)
+ do
+ do_facet ost"$i" lctl set_param \
+ ost.OSS.ost_io.nrs_policies="tbf\ jobid"
+ [ $? -ne 0 ] &&
+ error "failed to set TBF policy"
+ done
+
+ # Add a rule that only valid for Jobid TBF. If direct change between
+ # TBF types is not supported, this operation will fail.
+ tbf_rule_operate ost0 "start\ dd_runas\ {dd.$RUNAS_ID}\ 50"
+
+ # Cleanup the TBF policy
+ for i in $(seq 1 $OSTCOUNT)
+ do
+ do_facet ost"$i" lctl set_param \
+ ost.OSS.ost_io.nrs_policies="fifo"
+ [ $? -ne 0 ] &&
+ error "failed to set policy back to fifo"
+ done
+ return 0
+}
+run_test 77g "Change TBF type directly"
+
+test_80() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
+ local MDTIDX=1
+ local mdt_index
+ local i
+ local file
+ local pid
+
+ mkdir -p $DIR1/$tdir/dir
+ createmany -o $DIR1/$tdir/dir/f 10 ||
+ error "create files under remote dir failed $i"
+
+ cp /etc/passwd $DIR1/$tdir/$tfile
+
+ #migrate open file should fails
+ multiop_bg_pause $DIR2/$tdir/$tfile O_c || error "open $file failed"
+ pid=$!
+ # give multiop a chance to open
+ sleep 1
+
+ $LFS migrate -m $MDTIDX $DIR1/$tdir &&
+ error "migrate open files should failed with open files"
+
+ kill -USR1 $pid
+
+ $LFS migrate -m $MDTIDX $DIR1/$tdir ||
+ error "migrate remote dir error"
+
+ echo "Finish migration, then checking.."
+ for file in $(find $DIR1/$tdir); do
+ mdt_index=$($LFS getstripe -M $file)
+ [ $mdt_index == $MDTIDX ] ||
+ error "$file is not on MDT${MDTIDX}"
+ done
+
+ diff /etc/passwd $DIR1/$tdir/$tfile ||
+ error "file different after migration"
+
+ rm -rf $DIR1/$tdir || error "rm dir failed after migration"
+}
+run_test 80 "migrate directory when some children is being opened"
+
+test_81() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
+
+ rm -rf $DIR1/$tdir
+
+ mkdir -p $DIR1/$tdir
+
+ $LFS setdirstripe -i0 -c$MDSCOUNT $DIR1/$tdir/d0
+ $LFS setdirstripe -i0 -c$MDSCOUNT $DIR1/$tdir/d1
+
+ cd $DIR1/$tdir
+ touch d0/0 || error "create 0 failed"
+ mv d0/0 d1/0 || error "rename d0/0 d1/0 failed"
+ stat d0/0 && error "stat mv filed succeed"
+ mv $DIR2/$tdir/d1/0 $DIR2/$tdir/d0/0 || "rename d1/0 d0/0 failed"
+ stat d0/0 || error "stat failed"
+
+ local t=$(ls -ai $DIR1/$tdir/d0 | sort -u | wc -l)
+
+ if [ $t -ne 3 ]; then
+ ls -ai $DIR1/$tdir/d0
+ error "expect 3 get $t"
+ fi
+
+ return 0
+}
+run_test 81 "rename and stat under striped directory"
+
+test_82() {
+ [[ $(lustre_version_code $SINGLEMDS) -gt $(version_code 2.6.91) ]] ||
+ { skip "Need MDS version at least 2.6.92"; return 0; }
+
+ # Client 1 creates a file.
+ multiop_bg_pause $DIR1/$tfile O_ac || error "multiop_bg_pause 1"
+ pid1=$!
+ # Client 2 opens the file.
+ multiop_bg_pause $DIR2/$tfile o_Ac || error "multiop_bg_pause 2"
+ pid2=$!
+ # Client 1 makes the file an orphan.
+ rm $DIR1/$tfile || error "rm"
+ # Client 2 sets EA "user.multiop".
+ kill -s USR1 $pid2
+ wait $pid2 || error "multiop 2"
+ # Client 1 gets EA "user.multiop". This used to fail because the EA
+ # cache refill would get "trusted.link" from mdd_xattr_list() but
+ # -ENOENT when trying to get "trusted.link"'s value. See also sanity
+ # 102q.
+ kill -s USR1 $pid1
+ wait $pid1 || error "multiop 1"
+}
+run_test 82 "fsetxattr and fgetxattr on orphan files"
+