From: Jian Yu Date: Fri, 18 Dec 2015 08:35:10 +0000 (-0800) Subject: LU-7192 tests: conf-sanity test 32c supports DNE on ZFS X-Git-Tag: 2.7.65~54 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=e414f14d88a059b7bc99b730907060d72e951b03;p=fs%2Flustre-release.git LU-7192 tests: conf-sanity test 32c supports DNE on ZFS 1. ZFS needs import pool before tunefs. 2. previous 2.4 ZFS disk image contains 100M file, which will cause 'migrate' used up all OST space and fail, recreate the image. 3. generate test images for 2.7-formatted ldiskfs and zfs with DNE striped directories feature enabled and update t32_test() to test that. Test-Parameters: alwaysuploadlogs envdefinitions=SLOW=yes,ENABLE_QUOTA=yes,ONLY=32 mdtfilesystemtype=zfs mdsfilesystemtype=zfs ostfilesystemtype=zfs mdtcount=4 testlist=conf-sanity Change-Id: Ib35e4cb80d6beee9695b26844c4e6a99e88adb89 Signed-off-by: Lai Siyao Signed-off-by: Jian Yu Reviewed-on: http://review.whamcloud.com/16663 Tested-by: Jenkins Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Niu Yawei --- diff --git a/lustre/tests/Makefile.am b/lustre/tests/Makefile.am index 077c681..e98bb84 100644 --- a/lustre/tests/Makefile.am +++ b/lustre/tests/Makefile.am @@ -10,7 +10,7 @@ DIST_SUBDIRS = mpi noinst_DATA = disk1_8_up_2_5-ldiskfs.tar.bz2 disk2_1_up_2_5-ldiskfs.tar.bz2 noinst_DATA += disk2_3_up_2_5-ldiskfs.tar.bz2 disk2_4-ldiskfs.tar.bz2 -noinst_DATA += disk2_4-zfs.tar.bz2 +noinst_DATA += disk2_4-zfs.tar.bz2 disk2_7-ldiskfs.tar.bz2 disk2_7-zfs.tar.bz2 noinst_SCRIPTS = leak_finder.pl llmount.sh llmountcleanup.sh functions.sh noinst_SCRIPTS += test-framework.sh runvmstat runiozone runtests sanity.sh noinst_SCRIPTS += rundbench acceptance-small.sh compile.sh conf-sanity.sh diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index 4d64693..0d131e1 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -1332,8 +1332,8 @@ run_test 31 "Connect to non-existent node (shouldn't crash)" T32_QID=60000 -T32_BLIMIT=20480 # Kbytes -T32_ILIMIT=2 +T32_BLIMIT=40960 # Kbytes +T32_ILIMIT=4 # # This is not really a test but a tool to create new disk @@ -1349,6 +1349,10 @@ test_32newtarball() { local dst=. local src=/etc/rc.d local tmp=$TMP/t32_image_create + local server_version=$(lustre_version_code $SINGLEMDS) + local remote_dir + local striped_dir + local pushd_dir if [ $FSNAME != t32fs -o \( -z "$MDSDEV" -a -z "$MDSDEV1" \) -o \ $OSTCOUNT -ne 1 -o -z "$OSTDEV1" ]; then @@ -1367,7 +1371,7 @@ test_32newtarball() { mkdir $tmp/src || return 1 tar cf - -C $src . | tar xf - -C $tmp/src dd if=/dev/zero of=$tmp/src/t32_qf_old bs=1M \ - count=$(($T32_BLIMIT / 1024 / 2)) + count=$(($T32_BLIMIT / 1024 / 4)) chown $T32_QID.$T32_QID $tmp/src/t32_qf_old # format ost with comma-separated NIDs to verify LU-4460 @@ -1376,18 +1380,42 @@ test_32newtarball() { setupall - [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ] && + [[ $server_version -ge $(version_code 2.3.50) ]] || $LFS quotacheck -ug /mnt/$FSNAME $LFS setquota -u $T32_QID -b 0 -B $T32_BLIMIT -i 0 -I $T32_ILIMIT \ /mnt/$FSNAME tar cf - -C $tmp/src . | tar xf - -C /mnt/$FSNAME + + if [[ $MDSCOUNT -ge 2 ]]; then + remote_dir=/mnt/$FSNAME/remote_dir + $LFS mkdir -i 1 $remote_dir + tar cf - -C $tmp/src . | tar xf - -C $remote_dir + + if [[ $server_version -ge $(version_code 2.7.0) ]]; then + striped_dir=/mnt/$FSNAME/striped_dir_old + $LFS mkdir -i 1 -c 2 $striped_dir + tar cf - -C $tmp/src . | tar xf - -C $striped_dir + fi + fi + stopall mkdir $tmp/img || return 1 setupall - pushd /mnt/$FSNAME + + pushd_dir=/mnt/$FSNAME + if [[ $MDSCOUNT -ge 2 ]]; then + pushd_dir=$remote_dir + if [[ $server_version -ge $(version_code 2.7.0) ]]; then + pushd $striped_dir + ls -Rni --time-style=+%s >$tmp/img/list2 + popd + fi + fi + + pushd $pushd_dir ls -Rni --time-style=+%s >$tmp/img/list find ! -name .lustre -type f -exec sha1sum {} \; | sort -k 2 >$tmp/img/sha1sums @@ -1395,7 +1423,7 @@ test_32newtarball() { $LCTL get_param -n version | head -n 1 | sed -e 's/^lustre: *//' >$tmp/img/commit - [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ] && + [[ $server_version -ge $(version_code 2.3.50) ]] || $LFS quotaon -ug /mnt/$FSNAME $LFS quota -u $T32_QID -v /mnt/$FSNAME $LFS quota -v -u $T32_QID /mnt/$FSNAME | @@ -1406,11 +1434,8 @@ test_32newtarball() { awk 'BEGIN { num='5' } { if ($1 == "'/mnt/$FSNAME'") \ { if (NF == 1) { getline } else { num++ } ; print $num;} }' \ | tr -d "*" > $tmp/img/ispace - - if [ $MDSCOUNT -ge 2 ]; then - $LFS mkdir -i 1 /mnt/$FSNAME/remote_dir - tar cf - -C $tmp/src . | tar xf - -C /mnt/$FSNAME/remote_dir - fi + echo $T32_BLIMIT > $tmp/img/blimit + echo $T32_ILIMIT > $tmp/img/ilimit stopall @@ -1428,6 +1453,9 @@ test_32newtarball() { mv ${MDSDEV1:-$MDSDEV} $tmp/img for num in $(seq 2 $MDSCOUNT); do local devname=$(mdsdevname $num) + local facet=mds$num + [[ $(facet_fstype $facet) != zfs ]] || + devname=$(mdsvdevname $num) mv $devname $tmp/img done mv $OSTDEV1 $tmp/img @@ -1473,7 +1501,8 @@ t32_check() { t32_test_cleanup() { local tmp=$TMP/t32 - local fstype=$(facet_fstype $SINGLEMDS) + local facet=$SINGLEMDS + local fstype=$(facet_fstype $facet) local rc=$? if $shall_cleanup_lustre; then @@ -1491,9 +1520,15 @@ t32_test_cleanup() { $r rm -rf $tmp rm -rf $tmp - if [ $fstype == "zfs" ]; then - $r $ZPOOL destroy t32fs-mdt1 || rc=$? - $r $ZPOOL destroy t32fs-ost1 || rc=$? + if [[ $fstype == zfs ]]; then + local poolname + local poolname_list="t32fs-mdt1 t32fs-ost1" + + ! $mdt2_is_available || poolname_list+=" t32fs-mdt2" + + for poolname in $poolname_list; do + destroy_zpool $facet $poolname + done fi return $rc } @@ -1583,8 +1618,8 @@ t32_verify_quota() { awk 'BEGIN { num='3' } { if ($1 == "'$mnt'") \ { if (NF == 1) { getline } else { num++ } ; print $num;} }' \ | tr -d "*") - [ $qval -eq $T32_BLIMIT ] || { - echo "blimit, act:$qval, exp:$T32_BLIMIT" + [ $qval -eq $img_blimit ] || { + echo "blimit, act:$qval, exp:$img_blimit" return 1 } @@ -1592,8 +1627,8 @@ t32_verify_quota() { awk 'BEGIN { num='7' } { if ($1 == "'$mnt'") \ { if (NF == 1) { getline } else { num++ } ; print $num;} }' \ | tr -d "*") - [ $qval -eq $T32_ILIMIT ] || { - echo "ilimit, act:$qval, exp:$T32_ILIMIT" + [ $qval -eq $img_ilimit ] || { + echo "ilimit, act:$qval, exp:$img_ilimit" return 1 } @@ -1615,18 +1650,18 @@ t32_verify_quota() { chmod 0777 $mnt runas -u $T32_QID -g $T32_QID dd if=/dev/zero of=$mnt/t32_qf_new \ - bs=1M count=$(($T32_BLIMIT / 1024)) oflag=sync && { + bs=1M count=$((img_blimit / 1024)) oflag=sync && { echo "Write succeed, but expect -EDQUOT" return 1 } rm -f $mnt/t32_qf_new runas -u $T32_QID -g $T32_QID createmany -m $mnt/t32_qf_ \ - $T32_ILIMIT && { + $img_ilimit && { echo "Create succeed, but expect -EDQUOT" return 1 } - unlinkmany $mnt/t32_qf_ $T32_ILIMIT + unlinkmany $mnt/t32_qf_ $img_ilimit return 0 } @@ -1640,6 +1675,7 @@ t32_test() { local shall_cleanup_mdt1=false local shall_cleanup_ost=false local shall_cleanup_lustre=false + local mdt2_is_available=false local node=$(facet_active_host $SINGLEMDS) local r="do_node $node" local node2=$(facet_active_host mds2) @@ -1649,6 +1685,8 @@ t32_test() { local img_arch local img_bspace local img_ispace + local img_blimit + local img_ilimit local fsname=t32fs local nid=$($r $LCTL list_nids | head -1) local mopts @@ -1661,6 +1699,7 @@ t32_test() { local mdt2_dev=$tmp/mdt2 local ost_dev=$tmp/ost local stripe_index + local stripe_count local dir trap 'trap - RETURN; t32_test_cleanup' RETURN @@ -1677,6 +1716,14 @@ t32_test() { img_arch=$($r cat $tmp/arch) img_bspace=$($r cat $tmp/bspace) img_ispace=$($r cat $tmp/ispace) + + # older images did not have "blimit" and "ilimit" files + # use old values for T32_BLIMIT and T32_ILIMIT + $r test -f $tmp/blimit && img_blimit=$($r cat $tmp/blimit) || + img_blimit=20480 + $r test -f $tmp/ilimit && img_ilimit=$($r cat $tmp/ilimit) || + img_ilimit=2 + echo "Upgrading from $(basename $tarball), created with:" echo " Commit: $img_commit" echo " Kernel: $img_kernel" @@ -1691,12 +1738,23 @@ t32_test() { $(lustre_version_code ost1) -lt $(version_code 2.5.0) ] && ff_convert="no" - if [ $fstype == "zfs" ]; then + ! $r test -f $mdt2_dev || mdt2_is_available=true + + if [[ $fstype == zfs ]]; then # import pool first - $r $ZPOOL import -f -d $tmp t32fs-mdt1 - $r $ZPOOL import -f -d $tmp t32fs-ost1 + local poolname + local poolname_list="t32fs-mdt1 t32fs-ost1" + + ! $mdt2_is_available || poolname_list+=" t32fs-mdt2" + + for poolname in $poolname_list; do + $r "$ZPOOL list -H $poolname >/dev/null 2>&1 || + $ZPOOL import -f -d $tmp $poolname" + done + mdt_dev=t32fs-mdt1/mdt1 ost_dev=t32fs-ost1/ost1 + ! $mdt2_is_available || mdt2_dev=t32fs-mdt2/mdt2 wait_update_facet $SINGLEMDS "$ZPOOL list | awk '/^t32fs-mdt1/ { print \\\$1 }'" "t32fs-mdt1" || { error_noexit "import zfs pool failed" @@ -1712,7 +1770,7 @@ t32_test() { return 1 } - if $r test -f $mdt2_dev; then + if $mdt2_is_available; then $r $TUNEFS --dryrun $mdt2_dev || { $r losetup -a error_noexit "tunefs.lustre before mounting the MDT" @@ -1729,7 +1787,7 @@ t32_test() { error_noexit "Enable mdt quota feature" return 1 } - if $r test -f $mdt2_dev; then + if $mdt2_is_available; then $r $TUNEFS --quota $mdt2_dev || { $r losetup -a error_noexit "Enable mdt quota feature" @@ -1771,7 +1829,7 @@ t32_test() { } shall_cleanup_mdt=true - if $r test -f $mdt2_dev; then + if $mdt2_is_available; then mopts=mgsnode=$nid,$mopts $r $MOUNT_CMD -o $mopts $mdt2_dev $tmp/mnt/mdt1 || { $r losetup -a @@ -1799,6 +1857,8 @@ t32_test() { return 1 } + [[ $(facet_fstype mds1) != zfs ]] || import_zpool fs2mds + $r $TUNEFS --dryrun $fs2mdsdev || { error_noexit "tunefs.lustre before mounting the MDT" return 1 @@ -1943,9 +2003,59 @@ t32_test() { return 1 } + if $r test -f $tmp/list; then + # + # There is not a Test Framework API to copy files to or + # from a remote node. + # + # LU-2393 - do both sorts on same node to ensure locale + # is identical + local list_file=$tmp/list + + if $mdt2_is_available; then + if [[ -d $tmp/mnt/lustre/striped_dir_old ]] && + $r test -f $tmp/list2; then + list_file=$tmp/list2 + pushd $tmp/mnt/lustre/striped_dir_old + else + pushd $tmp/mnt/lustre/remote_dir + fi + else + pushd $tmp/mnt/lustre + fi + $r cat $list_file | sort -k 6 >$tmp/list.orig + ls -Rni --time-style=+%s | sort -k 6 >$tmp/list || { + error_noexit "ls" + return 1 + } + popd + # + # 32-bit and 64-bit clients use different algorithms to + # convert FIDs into inode numbers. Hence, remove the + # inode numbers from the lists, if the original list was + # created on an architecture with different number of + # bits per "long". + # + if [ $(t32_bits_per_long $(uname -m)) != \ + $(t32_bits_per_long $img_arch) ]; then + echo "Different number of bits per \"long\"" \ + "from the disk image" + for list in list.orig list; do + sed -i -e 's/^[0-9]\+[ \t]\+//' \ + $tmp/$list + done + fi + if ! diff -ub $tmp/list.orig $tmp/list; then + error_noexit "list verification failed" + return 1 + fi + else + echo "list verification skipped" + fi + if [ "$dne_upgrade" != "no" ]; then $LFS mkdir -i 1 -c2 $tmp/mnt/lustre/striped_dir || { - error_noexit "set remote dir failed" + error_noexit "set striped dir failed" return 1 } @@ -1953,9 +2063,10 @@ t32_test() { pushd $tmp/mnt/lustre tar -cf - . --exclude=./striped_dir \ + --exclude=./striped_dir_old \ --exclude=./remote_dir | tar -xvf - -C striped_dir 1>/dev/null || { - error_noexit "cp to remote dir failed" + error_noexit "cp to striped dir failed" return 1 } popd @@ -1963,20 +2074,42 @@ t32_test() { # If it is upgrade from DNE (2.5), then rename the remote dir, # which is created in 2.5 to striped dir. - if $r test -f $mdt2_dev; then - stripe_index=$(LFS getdirstripe -i \ + if $mdt2_is_available && [[ "$dne_upgrade" != "no" ]]; then + stripe_index=$($LFS getdirstripe -i \ $tmp/mnt/lustre/remote_dir) - [ $stripe_index -eq 1 ] || { - error_noexit "get index $striped_index failed" + + [[ $stripe_index -eq 1 ]] || { + error_noexit "get index \"$stripe_index\"" \ + "from remote dir failed" return 1 } mv $tmp/mnt/lustre/remote_dir \ $tmp/mnt/lustre/striped_dir/ || { - error_noexit "mv failed" + error_noexit "mv remote dir failed" + return 1 + } + fi + + # If it is upgraded from DNE (2.7), then move the striped dir + # which was created in 2.7 to the new striped dir. + if $mdt2_is_available && [[ "$dne_upgrade" != "no" ]] && + [[ -d $tmp/mnt/lustre/striped_dir_old ]]; then + stripe_count=$($LFS getdirstripe -c \ + $tmp/mnt/lustre/striped_dir_old) + [[ $stripe_count -eq 2 ]] || { + error_noexit "get count $stripe_count" \ + "from striped dir failed" + return 1 + } + mv $tmp/mnt/lustre/striped_dir_old \ + $tmp/mnt/lustre/striped_dir/ || { + error_noexit "mv striped dir failed" return 1 } fi + sync; sleep 5; sync + $r $LCTL set_param -n osd*.*.force_sync=1 dd if=/dev/zero of=$tmp/mnt/lustre/tmp_file bs=10k count=10 || { error_noexit "dd failed" return 1 @@ -1996,8 +2129,8 @@ t32_test() { pushd $tmp/mnt/lustre fi - find ! -path "*remote_dir*" ! -name .lustre -type f \ - -exec sha1sum {} \; | + find ! -path "*remote_dir*" ! -path "*striped_dir*" \ + ! -name .lustre -type f -exec sha1sum {} \; | sort -k 2 >$tmp/sha1sums || { popd error_noexit "sha1sum" @@ -2009,22 +2142,34 @@ t32_test() { return 1 fi - # if upgrade from DNE (2.5), then check remote directory - if $r test -f $mdt2_dev; then - pushd $tmp/mnt/lustre/striped_dir/remote_dir - find ! -name .lustre -type f \ - -exec sha1sum {} \; | - sort -k 2 >$tmp/sha1sums || { + # if upgrade from DNE(2.5), then check remote directory + # if upgrade from DNE(2.7), then check striped directory + if $mdt2_is_available && + [[ "$dne_upgrade" != "no" ]]; then + local new_dir="$tmp/mnt/lustre/striped_dir" + local striped_dir_old="$new_dir/striped_dir_old" + + local dir_list="$new_dir/remote_dir" + [[ ! -d $triped_dir_old ]] || + dir_list+=" $striped_dir_old" + + for dir in $dir_list; do + pushd $dir + find ! -name .lustre -type f \ + -exec sha1sum {} \; | + sort -k 2 >$tmp/sha1sums || { + popd + error_noexit "sha1sum" + return 1 + } popd - error_noexit "sha1sum" - return 1 - } - popd - if ! diff -ub $tmp/sha1sums.orig \ - $tmp/sha1sums; then - error_noexit "sha1sum dne failed" - return 1 - fi + if ! diff -ub $tmp/sha1sums.orig \ + $tmp/sha1sums; then + error_noexit "sha1sum $dir" \ + "failed" + return 1 + fi + done fi else echo "sha1sum verification skipped" @@ -2037,42 +2182,6 @@ t32_test() { } fi - if $r test -f $tmp/list; then - # - # There is not a Test Framework API to copy files to or - # from a remote node. - # - # LU-2393 - do both sorts on same node to ensure locale - # is identical - $r cat $tmp/list | sort -k 6 >$tmp/list.orig - pushd $tmp/mnt/lustre - ls -Rni --time-style=+%s | sort -k 6 >$tmp/list || { - error_noexit "ls" - return 1 - } - popd - # - # 32-bit and 64-bit clients use different algorithms to - # convert FIDs into inode numbers. Hence, remove the inode - # numbers from the lists, if the original list was created - # on an architecture with different number of bits per - # "long". - # - if [ $(t32_bits_per_long $(uname -m)) != \ - $(t32_bits_per_long $img_arch) ]; then - echo "Different number of bits per \"long\" from the disk image" - for list in list.orig list; do - sed -i -e 's/^[0-9]\+[ \t]\+//' $tmp/$list - done - fi - if ! diff -ub $tmp/list.orig $tmp/list; then - error_noexit "list verification failed" - return 1 - fi - else - echo "list verification skipped" - fi - # migrate files/dirs to remote MDT, then move them back if [ $(lustre_version_code mds1) -ge $(version_code 2.7.50) -a \ $dne_upgrade != "no" ]; then @@ -2136,7 +2245,7 @@ t32_test() { } shall_cleanup_lustre=false else - if [ "$dne_upgrade" != "no" ]; then + if [[ "$dne_upgrade" != "no" ]] || $mdt2_is_available; then $r $UMOUNT $tmp/mnt/mdt1 || { error_noexit "Unmounting the MDT2" return 1 @@ -2161,6 +2270,12 @@ t32_test() { return 1 } + if [[ $fstype == zfs ]]; then + local poolname=t32fs-mdt1 + $r "$ZPOOL list -H $poolname >/dev/null 2>&1 || + $ZPOOL import -f -d $tmp $poolname" + fi + # mount a second time to make sure we didnt leave upgrade flag on $r $TUNEFS --dryrun $mdt_dev || { $r losetup -a diff --git a/lustre/tests/disk2_4-zfs.tar.bz2 b/lustre/tests/disk2_4-zfs.tar.bz2 index 289d7de..15c6dff 100644 Binary files a/lustre/tests/disk2_4-zfs.tar.bz2 and b/lustre/tests/disk2_4-zfs.tar.bz2 differ diff --git a/lustre/tests/disk2_7-ldiskfs.tar.bz2 b/lustre/tests/disk2_7-ldiskfs.tar.bz2 new file mode 100644 index 0000000..b5a6974 Binary files /dev/null and b/lustre/tests/disk2_7-ldiskfs.tar.bz2 differ diff --git a/lustre/tests/disk2_7-zfs.tar.bz2 b/lustre/tests/disk2_7-zfs.tar.bz2 new file mode 100644 index 0000000..62ed6e7 Binary files /dev/null and b/lustre/tests/disk2_7-zfs.tar.bz2 differ