X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=2a62f83c99f442b5cb00f650fe5b164c55a24a17;hb=9e34b1b7fb178b248cb6fa64ac2799dcce8aeb47;hp=231b6c346ef12d5ec300ce2642b0fd09e932a130;hpb=d2d68b18396c95dd273f6c6d8e11f080a9ac3051;p=fs%2Flustre-release.git diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 231b6c3..2a62f83 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -5,6 +5,7 @@ trap 'print_summary && touch $TF_FAIL && \ set -e #set -x +export LANG=en_US export EJOURNAL=${EJOURNAL:-""} export REFORMAT=${REFORMAT:-""} export WRITECONF=${WRITECONF:-""} @@ -178,6 +179,15 @@ init_test_env() { fi fi + export RESIZE2FS=$RESIZE2FS + if [ -z "$RESIZE2FS" ]; then + if which resizefs.ldiskfs >/dev/null 2>&1; then + export RESIZE2FS=resizefs.ldiskfs + else + export RESIZE2FS=resize2fs + fi + fi + export LFSCK_BIN=${LFSCK_BIN:-lfsck} export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after each test suite export FSCK_MAX_ERR=4 # File system errors left uncorrected @@ -291,13 +301,15 @@ init_test_env() { export RLUSTRE=${RLUSTRE:-$LUSTRE} export RPWD=${RPWD:-$PWD} export I_MOUNTED=${I_MOUNTED:-"no"} - if [ ! -f /lib/modules/$(uname -r)/kernel/fs/lustre/mdt.ko -a \ - ! -f /lib/modules/$(uname -r)/updates/kernel/fs/lustre/mdt.ko -a \ - ! -f `dirname $0`/../mdt/mdt.ko ]; then - export CLIENTMODSONLY=yes - fi + if [ ! -f /lib/modules/$(uname -r)/kernel/fs/lustre/mdt.ko -a \ + ! -f /lib/modules/$(uname -r)/updates/kernel/fs/lustre/mdt.ko -a \ + ! -f /lib/modules/$(uname -r)/extra/kernel/fs/lustre/mdt.ko -a \ + ! -f $LUSTRE/mdt/mdt.ko ]; then + export CLIENTMODSONLY=yes + fi - export SHUTDOWN_ATTEMPTS=${SHUTDOWN_ATTEMPTS:-3} + export SHUTDOWN_ATTEMPTS=${SHUTDOWN_ATTEMPTS:-3} + export OSD_TRACK_DECLARES_LBUG=${OSD_TRACK_DECLARES_LBUG:-"yes"} # command line @@ -474,7 +486,6 @@ load_modules_local() { load_module ../lnet/lnet/lnet LNETLND=${LNETLND:-"socklnd/ksocklnd"} load_module ../lnet/klnds/$LNETLND - load_module lvfs/lvfs load_module obdclass/obdclass load_module ptlrpc/ptlrpc load_module ptlrpc/gss/ptlrpc_gss @@ -491,24 +502,20 @@ load_modules_local() { grep -q crc16 $SYMLIST || { modprobe crc16 2>/dev/null || true; } grep -q -w jbd $SYMLIST || { modprobe jbd 2>/dev/null || true; } grep -q -w jbd2 $SYMLIST || { modprobe jbd2 2>/dev/null || true; } + load_module lfsck/lfsck [ "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS if [[ $(node_fstypes $HOSTNAME) == *zfs* ]]; then modprobe zfs load_module osd-zfs/osd_zfs fi - load_module mgs/mgs - load_module mdd/mdd if [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then - # - # This block shall be moved up beside osd-zfs as soon - # as osd-ldiskfs stops using mdd symbols. - # grep -q exportfs_decode_fh $SYMLIST || { modprobe exportfs 2> /dev/null || true; } - load_module ../ldiskfs/ldiskfs/ldiskfs - load_module lvfs/fsfilt_ldiskfs + load_module ../ldiskfs/ldiskfs load_module osd-ldiskfs/osd_ldiskfs fi + load_module mgs/mgs + load_module mdd/mdd load_module mdt/mdt load_module ost/ost load_module lod/lod @@ -592,7 +599,7 @@ fs_log_size() { local size=0 case $fstype in ldiskfs) size=50;; # largest seen is 44, leave some headroom - zfs) size=256;; + zfs) size=400;; # largest seen is 384 esac echo -n $size @@ -710,6 +717,13 @@ cleanup_gss() { fi } +facet_svc() { + local facet=$1 + local var=${facet}_svc + + echo -n ${!var} +} + facet_type() { local facet=$1 @@ -886,6 +900,33 @@ zpool_name() { } # +# Create ZFS storage pool. +# +create_zpool() { + local facet=$1 + local poolname=$2 + local vdev=$3 + shift 3 + local opts=${@:-"-o cachefile=none"} + + do_facet $facet "$ZPOOL list -H $poolname >/dev/null 2>&1 || + $ZPOOL create -f $opts $poolname $vdev" +} + +# +# Create ZFS file system. +# +create_zfs() { + local facet=$1 + local dataset=$2 + shift 2 + local opts=${@:-"-o mountpoint=legacy"} + + do_facet $facet "$ZFS list -H $dataset >/dev/null 2>&1 || + $ZFS create $opts $dataset" +} + +# # Export ZFS storage pool. # Before exporting the pool, all datasets within the pool should be unmounted. # @@ -899,11 +940,28 @@ export_zpool() { if [[ -n "$poolname" ]]; then do_facet $facet "! $ZPOOL list -H $poolname >/dev/null 2>&1 || + grep -q ^$poolname/ /proc/mounts || $ZPOOL export $opts $poolname" fi } # +# Destroy ZFS storage pool. +# Destroy the given pool and free up any devices for other use. This command +# tries to unmount any active datasets before destroying the pool. +# -f Force any active datasets contained within the pool to be unmounted. +# +destroy_zpool() { + local facet=$1 + local poolname=${2:-$(zpool_name $facet)} + + if [[ -n "$poolname" ]]; then + do_facet $facet "! $ZPOOL list -H $poolname >/dev/null 2>&1 || + $ZPOOL destroy -f $poolname" + fi +} + +# # Import ZFS storage pool. # Force importing, even if the pool appears to be potentially active. # @@ -916,7 +974,9 @@ import_zpool() { poolname=$(zpool_name $facet) if [[ -n "$poolname" ]]; then - do_facet $facet "$ZPOOL import -f $opts $poolname" + opts+=" -d $(dirname $(facet_vdevice $facet))" + do_facet $facet "$ZPOOL list -H $poolname >/dev/null 2>&1 || + $ZPOOL import -f $opts $poolname" fi } @@ -1285,6 +1345,23 @@ lfs_df() { $LFS df $* | sed -e 's/filesystem /filesystem_/' } +# Get free inodes on the MDT specified by mdt index, free indoes on +# the whole filesystem will be returned when index == -1. +mdt_free_inodes() { + local index=$1 + local free_inodes + local mdt_uuid + + if [ $index -eq -1 ]; then + mdt_uuid="summary" + else + mdt_uuid=$(mdtuuid_from_index $index) + fi + + free_inodes=$(lfs_df -i $MOUNT | grep $mdt_uuid | awk '{print $4}') + echo $free_inodes +} + setup_quota(){ if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then setup_quota_old $1 @@ -1659,6 +1736,7 @@ TESTLOG_PREFIX=$TESTLOG_PREFIX \ TESTNAME=$TESTNAME \ DBENCH_LIB=$DBENCH_LIB \ DBENCH_SRC=$DBENCH_SRC \ +CLIENT_COUNT=$((CLIENTCOUNT - 1)) \ LFS=$LFS \ run_${load}.sh" & local ppid=$! @@ -1868,30 +1946,43 @@ cleanup_check() { } wait_update () { - local node=$1 - local TEST=$2 - local FINAL=$3 - local MAX=${4:-90} - - local RESULT - local WAIT=0 - local sleep=1 - local print=10 - while [ true ]; do - RESULT=$(do_node $node "$TEST") - if [ "$RESULT" == "$FINAL" ]; then - [ -z "$RESULT" -o $WAIT -le $sleep ] || - echo "Updated after ${WAIT}s: wanted '$FINAL' got '$RESULT'" - return 0 - fi - [ $WAIT -ge $MAX ] && break - [ $((WAIT % print)) -eq 0 ] && - echo "Waiting $((MAX - WAIT)) secs for update" - WAIT=$((WAIT + sleep)) - sleep $sleep - done - echo "Update not seen after ${MAX}s: wanted '$FINAL' got '$RESULT'" - return 3 + local verbose=false + if [[ "$1" == "--verbose" ]]; then + shift + verbose=true + fi + + local node=$1 + local TEST=$2 + local FINAL=$3 + local MAX=${4:-90} + local RESULT + local PREV_RESULT + local WAIT=0 + local sleep=1 + local print=10 + + while [ true ]; do + RESULT=$(do_node $node "$TEST") + if [[ "$RESULT" == "$FINAL" ]]; then + [[ -z "$RESULT" || $WAIT -le $sleep ]] || + echo "Updated after ${WAIT}s: wanted '$FINAL'"\ + "got '$RESULT'" + return 0 + fi + if [[ $verbose && "$RESULT" != "$PREV_RESULT" ]]; then + echo "Changed after ${WAIT}s: from '$PREV_RESULT'"\ + "to '$RESULT'" + PREV_RESULT=$RESULT + fi + [[ $WAIT -ge $MAX ]] && break + [[ $((WAIT % print)) -eq 0 ]] && + echo "Waiting $((MAX - WAIT)) secs for update" + WAIT=$((WAIT + sleep)) + sleep $sleep + done + echo "Update not seen after ${MAX}s: wanted '$FINAL' got '$RESULT'" + return 3 } wait_update_facet() { @@ -2025,6 +2116,7 @@ wait_mds_ost_sync () { echo "Waiting for orphan cleanup..." # MAX value includes time needed for MDS-OST reconnection local MAX=$(( TIMEOUT * 2 )) + local WAIT_TIMEOUT=${1:-$MAX} local WAIT=0 local new_wait=true local list=$(comma_list $(mdts_nodes)) @@ -2037,7 +2129,9 @@ wait_mds_ost_sync () { list=$(comma_list $(osts_nodes)) cmd="$LCTL get_param -n obdfilter.*.mds_sync" fi - while [ $WAIT -lt $MAX ]; do + + echo "wait $WAIT_TIMEOUT secs maximumly for $list mds-ost sync done." + while [ $WAIT -lt $WAIT_TIMEOUT ]; do local -a sync=($(do_nodes $list "$cmd")) local con=1 local i @@ -2053,10 +2147,13 @@ wait_mds_ost_sync () { done sleep 2 # increase waiting time and cover statfs cache [ ${con} -eq 1 ] && return 0 - echo "Waiting $WAIT secs for $facet mds-ost sync done." + echo "Waiting $WAIT secs for $list $i mds-ost sync done." WAIT=$((WAIT + 2)) done - echo "$facet recovery not done in $MAX sec. $STATUS" + + # show which nodes are not finished. + do_nodes $list "$cmd" + echo "$facet recovery node $i not done in $WAIT_TIMEOUT sec. $STATUS" return 1 } @@ -2308,14 +2405,32 @@ replay_barrier_nosync() { $LCTL mark "local REPLAY BARRIER on ${!svc}" } +# +# Get Lustre client uuid for a given Lustre mount point. +# +get_client_uuid() { + local mntpnt=${1:-$MOUNT} + + local name=$($LFS getname $mntpnt | cut -d' ' -f1) + local uuid=$($LCTL get_param -n llite.$name.uuid) + + echo -n $uuid +} + mds_evict_client() { - UUID=`lctl get_param -n mdc.${mds1_svc}-mdc-*.uuid` - do_facet mds1 "lctl set_param -n mdt.${mds1_svc}.evict_client $UUID" + local mntpnt=${1:-$MOUNT} + local uuid=$(get_client_uuid $mntpnt) + + do_facet $SINGLEMDS \ + "$LCTL set_param -n mdt.${mds1_svc}.evict_client $uuid" } ost_evict_client() { - UUID=`lctl get_param -n devices| grep ${ost1_svc}-osc- | egrep -v 'MDT' | awk '{print $5}'` - do_facet ost1 "lctl set_param -n obdfilter.${ost1_svc}.evict_client $UUID" + local mntpnt=${1:-$MOUNT} + local uuid=$(get_client_uuid $mntpnt) + + do_facet ost1 \ + "$LCTL set_param -n obdfilter.${ost1_svc}.evict_client $uuid" } fail() { @@ -2360,42 +2475,43 @@ host_nids_address() { } h2name_or_ip() { - if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else - echo $1"@$2" - fi + if [ "$1" = "'*'" ]; then echo \'*\'; else + echo $1"@$2" + fi } h2ptl() { - if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else - ID=`xtprocadmin -n $1 2>/dev/null | egrep -v 'NID' | awk '{print $1}'` - if [ -z "$ID" ]; then - echo "Could not get a ptl id for $1..." - exit 1 - fi - echo $ID"@ptl" - fi + if [ "$1" = "'*'" ]; then echo \'*\'; else + ID=`xtprocadmin -n $1 2>/dev/null | egrep -v 'NID' | \ + awk '{print $1}'` + if [ -z "$ID" ]; then + echo "Could not get a ptl id for $1..." + exit 1 + fi + echo $ID"@ptl" + fi } declare -fx h2ptl h2tcp() { - h2name_or_ip "$1" "tcp" + h2name_or_ip "$1" "tcp" } declare -fx h2tcp h2elan() { - if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else - if type __h2elan >/dev/null 2>&1; then - ID=$(__h2elan $1) - else - ID=`echo $1 | sed 's/[^0-9]*//g'` - fi - echo $ID"@elan" - fi + if [ "$1" = "'*'" ]; then echo \'*\'; else + if type __h2elan >/dev/null 2>&1; then + ID=$(__h2elan $1) + else + ID=`echo $1 | sed 's/[^0-9]*//g'` + fi + echo $ID"@elan" + fi } declare -fx h2elan h2o2ib() { - h2name_or_ip "$1" "o2ib" + h2name_or_ip "$1" "o2ib" } declare -fx h2o2ib @@ -2782,9 +2898,10 @@ add() { fi } +# Device formatted as ost ostdevname() { - num=$1 - DEVNAME=OSTDEV$num + local num=$1 + local DEVNAME=OSTDEV$num local fstype=$(facet_fstype ost$num) @@ -2793,8 +2910,9 @@ ostdevname() { #if $OSTDEVn isn't defined, default is $OSTDEVBASE + num eval DEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}};; zfs ) - #dataset name is independent of vdev device names - eval DEVPTR=${FSNAME}-ost${num}/ost${num};; + #try $OSTZFSDEVn - independent of vdev + DEVNAME=OSTZFSDEV$num + eval DEVPTR=${!DEVNAME:=${FSNAME}-ost${num}/ost${num}};; * ) error "unknown fstype!";; esac @@ -2802,9 +2920,11 @@ ostdevname() { echo -n $DEVPTR } +# Physical device location of data ostvdevname() { - num=$1 - DEVNAME=OSTDEV$num + local num=$1 + local DEVNAME + local VDEVPTR local fstype=$(facet_fstype ost$num) @@ -2813,7 +2933,9 @@ ostvdevname() { # vdevs are not supported by ldiskfs eval VDEVPTR="";; zfs ) - #if $OSTDEVn isn't defined, default is $OSTDEVBASE + num + #if $OSTDEVn isn't defined, default is $OSTDEVBASE{n} + # Device formated by zfs + DEVNAME=OSTDEV$num eval VDEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}};; * ) error "unknown fstype!";; @@ -2822,19 +2944,21 @@ ostvdevname() { echo -n $VDEVPTR } +# Logical device formated for lustre mdsdevname() { - num=$1 - DEVNAME=MDSDEV$num + local num=$1 + local DEVNAME=MDSDEV$num local fstype=$(facet_fstype mds$num) case $fstype in ldiskfs ) - #if $MDSDEVn isn't defined, default is $MDSDEVBASE + num + #if $MDSDEVn isn't defined, default is $MDSDEVBASE{n} eval DEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}};; zfs ) - #dataset name is independent of vdev device names - eval DEVPTR=${FSNAME}-mdt${num}/mdt${num};; + # try $MDSZFSDEVn - independent of vdev + DEVNAME=MDSZFSDEV$num + eval DEVPTR=${!DEVNAME:=${FSNAME}-mdt${num}/mdt${num}};; * ) error "unknown fstype!";; esac @@ -2842,10 +2966,10 @@ mdsdevname() { echo -n $DEVPTR } +# Physical location of data mdsvdevname() { - num=$1 - DEVNAME=MDSDEV$num - + local VDEVPTR="" + local num=$1 local fstype=$(facet_fstype mds$num) case $fstype in @@ -2853,7 +2977,9 @@ mdsvdevname() { # vdevs are not supported by ldiskfs eval VDEVPTR="";; zfs ) - #if $MDSDEVn isn't defined, default is $MDSDEVBASE + num + # if $MDSDEVn isn't defined, default is $MDSDEVBASE{n} + # Device formated by ZFS + local DEVNAME=MDSDEV$num eval VDEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}};; * ) error "unknown fstype!";; @@ -2876,10 +3002,11 @@ mgsdevname() { fi;; zfs ) if [ $(facet_host mgs) = $(facet_host mds1) ] && - ( [ -z "$MGSDEV" ] || [ $MGSDEV = $(mdsvdevname 1) ] ); then + ( [ -z "$MGSZFSDEV" ] && + [ -z "$MGSDEV" -o "$MGSDEV" = $(mdsvdevname 1) ] ); then DEVPTR=$(mdsdevname 1) else - DEVPTR=${FSNAME}-mgs/mgs + DEVPTR=${MGSZFSDEV:-${FSNAME}-mgs/mgs} fi;; * ) error "unknown fstype!";; @@ -2889,8 +3016,7 @@ mgsdevname() { } mgsvdevname() { - local VDEVPTR - DEVNAME=MGSDEV + local VDEVPTR="" local fstype=$(facet_fstype mgs) @@ -2900,9 +3026,10 @@ mgsvdevname() { ;; zfs ) if [ $(facet_host mgs) = $(facet_host mds1) ] && - ( [ -z "$MGSDEV" ] || [ $MGSDEV = $(mdsvdevname 1) ] ); then + ( [ -z "$MGSDEV" ] && + [ -z "$MGSZFSDEV" -o "$MGSZFSDEV" = $(mdsdevname 1) ]); then VDEVPTR=$(mdsvdevname 1) - else + elif [ -n "$MGSDEV" ]; then VDEVPTR=$MGSDEV fi;; * ) @@ -2922,6 +3049,84 @@ facet_mntpt () { echo -n $mntpt } +mount_ldiskfs() { + local facet=$1 + local dev=$(facet_device $facet) + local mnt=$(facet_mntpt $facet) + local opts + + if ! do_facet $facet test -b $dev; then + opts="-o loop" + fi + do_facet $facet mount -t ldiskfs $opts $dev $mnt +} + +unmount_ldiskfs() { + local facet=$1 + local dev=$(facet_device $facet) + local mnt=$(facet_mntpt $facet) + + do_facet $facet umount -d $mnt +} + +var_name() { + echo -n "$1" | tr -c '[:alnum:]\n' '_' +} + +mount_zfs() { + local facet=$1 + local ds=$(facet_device $facet) + local mnt=$(facet_mntpt $facet) + local canmnt + local mntpt + + import_zpool $facet + canmnt=$(do_facet $facet $ZFS get -H -o value canmount $ds) + mntpt=$(do_facet $facet $ZFS get -H -o value mountpoint $ds) + do_facet $facet $ZFS set canmount=noauto $ds + # + # The "legacy" mount method is used here because "zfs unmount $mnt" + # calls stat(2) on $mnt/../*, which may include $MOUNT. If certain + # targets are not available at the time, the stat(2) on $MOUNT will + # hang. + # + do_facet $facet $ZFS set mountpoint=legacy $ds + do_facet $facet mount -t zfs $ds $mnt + eval export mz_$(var_name ${facet}_$ds)_canmount=$canmnt + eval export mz_$(var_name ${facet}_$ds)_mountpoint=$mntpt +} + +unmount_zfs() { + local facet=$1 + local ds=$(facet_device $facet) + local mnt=$(facet_mntpt $facet) + local var_mntpt=mz_$(var_name ${facet}_$ds)_mountpoint + local var_canmnt=mz_$(var_name ${facet}_$ds)_canmount + local mntpt=${!var_mntpt} + local canmnt=${!var_canmnt} + + unset $var_mntpt + unset $var_canmnt + do_facet $facet umount $mnt + do_facet $facet $ZFS set mountpoint=$mntpt $ds + do_facet $facet $ZFS set canmount=$canmnt $ds + export_zpool $facet +} + +mount_fstype() { + local facet=$1 + local fstype=$(facet_fstype $facet) + + mount_$fstype $facet +} + +unmount_fstype() { + local facet=$1 + local fstype=$(facet_fstype $facet) + + unmount_$fstype $facet +} + ######## ## MountConf setup @@ -3528,10 +3733,17 @@ is_empty_dir() { # empty lustre filesystem may have empty directories lost+found and .lustre is_empty_fs() { + # exclude .lustre & lost+found [ $(find $1 -maxdepth 1 -name lost+found -o -name .lustre -prune -o \ -print | wc -l) = 1 ] || return 1 [ ! -d $1/lost+found ] || is_empty_dir $1/lost+found || return 1 - [ ! -d $1/.lustre ] || is_empty_dir $1/.lustre || return 1 + if [ $(lustre_version_code $SINGLEMDS) -gt $(version_code 2.4.0) ]; then + # exclude .lustre/fid (LU-2780) + [ $(find $1/.lustre -maxdepth 1 -name fid -prune -o \ + -print | wc -l) = 1 ] || return 1 + else + [ ! -d $1/.lustre ] || is_empty_dir $1/.lustre || return 1 + fi return 0 } @@ -3593,10 +3805,19 @@ check_and_setup_lustre() { set_default_debug_nodes $(comma_list $(nodes_list)) fi - if [ -n "$OSD_TRACK_DECLARES_LBUG" ] ; then - do_nodes $(comma_list $(mdts_nodes) $(osts_nodes)) \ - "$LCTL set_param osd-*.track_declares_assert=1" \ - > /dev/null + if [ $(lower $OSD_TRACK_DECLARES_LBUG) == 'yes' ] ; then + local facets="" + [ "$(facet_fstype ost1)" = "ldiskfs" ] && + facets="$(get_facets OST)" + [ "$(facet_fstype mds1)" = "ldiskfs" ] && + facets="$facets,$(get_facets MDS)" + [ "$(facet_fstype mgs)" = "ldiskfs" ] && + facets="$facets,mgs" + local nodes="$(facets_hosts ${facets})" + if [ -n "$nodes" ] ; then + do_nodes $nodes "$LCTL set_param \ + osd-ldiskfs.track_declares_assert=1 || true" + fi fi init_gss @@ -3687,13 +3908,27 @@ run_e2fsck() { return 0 } +# +# Run resize2fs on MDT or OST device. +# +run_resize2fs() { + local facet=$1 + local device=$2 + local size=$3 + shift 3 + local opts="$@" + + do_facet $facet "$RESIZE2FS $opts $device $size" +} + # verify a directory is shared among nodes. check_shared_dir() { local dir=$1 + local list=${2:-$(comma_list $(nodes_list))} [ -z "$dir" ] && return 1 - do_rpc_nodes "$(comma_list $(nodes_list))" check_logdir $dir - check_write_access $dir || return 1 + do_rpc_nodes "$list" check_logdir $dir + check_write_access $dir "$list" || return 1 return 0 } @@ -3928,7 +4163,7 @@ get_facets () { case $type in MGS ) list="$list $name";; - MDS|OST ) local count=${type}COUNT + MDS|OST|AGT ) local count=${type}COUNT for ((i=1; i<=${!count}; i++)) do list="$list ${name}$i" done;; @@ -4196,7 +4431,11 @@ error_noexit() { if [ -z "$*" ]; then echo "error() without useful message, please fix" > $LOGDIR/err else - echo "$@" > $LOGDIR/err + if [[ `echo $TYPE | grep ^IGNORE` ]]; then + echo "$@" > $LOGDIR/ignore + else + echo "$@" > $LOGDIR/err + fi fi } @@ -4219,9 +4458,10 @@ error_exit() { # use only if we are ignoring failures for this test, bugno required. # (like ALWAYS_EXCEPT, but run the test and ignore the results.) -# e.g. error_ignore 5494 "your message" +# e.g. error_ignore bz5494 "your message" or +# error_ignore LU-5494 "your message" error_ignore() { - local TYPE="IGNORE (bz$1)" + local TYPE="IGNORE ($1)" shift error_noexit "$@" } @@ -4475,6 +4715,7 @@ run_one_logged() { local name=${TESTSUITE}.test_${1}.test_log.$(hostname -s).log local test_log=$LOGDIR/$name rm -rf $LOGDIR/err + rm -rf $LOGDIR/ignore rm -rf $LOGDIR/skip local SAVE_UMASK=`umask` umask 0022 @@ -4484,7 +4725,7 @@ run_one_logged() { (run_one $1 "$2") 2>&1 | tee -i $test_log local RC=${PIPESTATUS[0]} - [ $RC -ne 0 ] && [ ! -f $LOGDIR/err ] && \ + [ $RC -ne 0 ] && [ ! -f $LOGDIR/err ] && echo "test_$1 returned $RC" | tee $LOGDIR/err duration=$((`date +%s` - $BEFORE)) @@ -4492,6 +4733,8 @@ run_one_logged() { if [[ -f $LOGDIR/err ]]; then TEST_ERROR=$(cat $LOGDIR/err) + elif [[ -f $LOGDIR/ignore ]]; then + TEST_ERROR=$(cat $LOGDIR/ignore) elif [[ -f $LOGDIR/skip ]]; then TEST_ERROR=$(cat $LOGDIR/skip) fi @@ -4719,6 +4962,11 @@ osts_nodes () { echo -n $(facets_nodes $(get_facets OST)) } +# Get all of the active AGT (HSM agent) nodes. +agts_nodes () { + echo -n $(facets_nodes $(get_facets AGT)) +} + # Get all of the client nodes and active server nodes. nodes_list () { local nodes=$HOSTNAME @@ -5183,7 +5431,7 @@ get_mds_dir () { mdsrate_cleanup () { if [ -d $4 ]; then - mpi_run -np $1 ${MACHINEFILE_OPTION} $2 ${MDSRATE} --unlink \ + mpi_run ${MACHINEFILE_OPTION} $2 -np $1 ${MDSRATE} --unlink \ --nfiles $3 --dir $4 --filefmt $5 $6 rmdir $4 fi @@ -5196,7 +5444,7 @@ delayed_recovery_enabled () { ######################## -convert_facet2label() { +convert_facet2label() { local facet=$1 if [ x$facet = xost ]; then @@ -5207,7 +5455,7 @@ convert_facet2label() { if [ -n ${!varsvc} ]; then echo ${!varsvc} - else + else error "No lablel for $facet!" fi } @@ -5967,19 +6215,20 @@ check_logdir() { } check_write_access() { - local dir=$1 - local node - local file + local dir=$1 + local list=${2:-$(comma_list $(nodes_list))} + local node + local file - for node in $(nodes_list); do - file=$dir/check_file.$(short_hostname $node) - if [[ ! -f "$file" ]]; then - # Logdir not accessible/writable from this node. - return 1 - fi - rm -f $file || return 1 - done - return 0 + for node in ${list//,/ }; do + file=$dir/check_file.$(short_nodename $node) + if [[ ! -f "$file" ]]; then + # Logdir not accessible/writable from this node. + return 1 + fi + rm -f $file || return 1 + done + return 0 } init_logging() { @@ -6060,24 +6309,23 @@ run_llverfs() #Remove objects from OST remove_ost_objects() { - shift - local ostdev=$1 - local group=$2 - shift 2 + local facet=$1 + local ostdev=$2 + local group=$3 + shift 3 local objids="$@" - local facet=ost$((OSTIDX + 1)) local mntpt=$(facet_mntpt $facet) local opts=$OST_MOUNT_OPTS local i local rc echo "removing objects from $ostdev on $facet: $objids" - if ! do_facet $facet test -b $ostdev; then + if ! test -b $ostdev; then opts=$(csa_add "$opts" -o loop) fi mount -t $(facet_fstype $facet) $opts $ostdev $mntpt || return $? - rc=0; + rc=0 for i in $objids; do rm $mntpt/O/$group/d$((i % 32))/$i || { rc=$?; break; } done @@ -6101,7 +6349,7 @@ remove_mdt_files() { fi mount -t $(facet_fstype $facet) $opts $mdtdev $mntpt || return $? - rc=0; + rc=0 for f in $files; do rm $mntpt/ROOT/$f || { rc=$?; break; } done @@ -6189,6 +6437,19 @@ min_ost_size () { $LCTL get_param -n osc.*.kbytesavail | sort -n | head -n1 } +# +# Get the block count of the filesystem. +# +get_block_count() { + local facet=$1 + local device=$2 + local count + + count=$(do_facet $facet "$DUMPE2FS -h $device 2>&1" | + awk '/^Block count:/ {print $3}') + echo -n $count +} + # Get the block size of the filesystem. get_block_size() { local facet=$1 @@ -6206,7 +6467,8 @@ large_xattr_enabled() { local mds_dev=$(mdsdevname ${SINGLEMDS//mds/}) - do_facet $SINGLEMDS "$DUMPE2FS -h $mds_dev 2>&1 | grep -q large_xattr" + do_facet $SINGLEMDS "$DUMPE2FS -h $mds_dev 2>&1 | + grep -E -q '(ea_inode|large_xattr)'" return ${PIPESTATUS[0]} } @@ -6245,30 +6507,33 @@ generate_string() { } reformat_external_journal() { + local facet=$1 + if [ ! -z ${EJOURNAL} ]; then - local rcmd="do_facet ${SINGLEMDS}" + local rcmd="do_facet $facet" - echo "reformat external journal on ${SINGLEMDS}:${EJOURNAL}" + echo "reformat external journal on $facet:${EJOURNAL}" ${rcmd} mke2fs -O journal_dev ${EJOURNAL} || return 1 fi } # MDT file-level backup/restore mds_backup_restore() { - local devname=$(mdsdevname ${SINGLEMDS//mds/}) + local facet=$1 + local igif=$2 + local devname=$(mdsdevname $(facet_number $facet)) local mntpt=$(facet_mntpt brpt) - local rcmd="do_facet ${SINGLEMDS}" + local rcmd="do_facet $facet" local metaea=${TMP}/backup_restore.ea local metadata=${TMP}/backup_restore.tgz local opts=${MDS_MOUNT_OPTS} - local svc=${SINGLEMDS}_svc - local igif=$1 + local svc=${facet}_svc if ! ${rcmd} test -b ${devname}; then opts=$(csa_add "$opts" -o loop) fi - echo "file-level backup/restore on ${SINGLEMDS}:${devname}" + echo "file-level backup/restore on $facet:${devname}" # step 1: build mount point ${rcmd} mkdir -p $mntpt @@ -6290,12 +6555,12 @@ mds_backup_restore() { # step 6: umount ${rcmd} umount -d $mntpt || return 4 # step 7: reformat external journal if needed - reformat_external_journal || return 5 + reformat_external_journal $facet || return 5 # step 8: reformat dev echo "reformat new device" - add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS} ${devname}) --backfstype \ - ldiskfs --reformat ${devname} $(mdsvdevname 1) > /dev/null || - exit 6 + add $facet $(mkfs_opts $facet ${devname}) --backfstype ldiskfs \ + --reformat ${devname} $(mdsvdevname $(facet_number $facet)) \ + > /dev/null || exit 6 # step 9: mount dev ${rcmd} mount -t ldiskfs $opts $devname $mntpt || return 7 # step 10: restore metadata @@ -6317,17 +6582,18 @@ mds_backup_restore() { # remove OI files mds_remove_ois() { - local devname=$(mdsdevname ${SINGLEMDS//mds/}) + local facet=$1 + local idx=$2 + local devname=$(mdsdevname $(facet_number $facet)) local mntpt=$(facet_mntpt brpt) - local rcmd="do_facet ${SINGLEMDS}" - local idx=$1 + local rcmd="do_facet $facet" local opts=${MDS_MOUNT_OPTS} if ! ${rcmd} test -b ${devname}; then opts=$(csa_add "$opts" -o loop) fi - echo "remove OI files: idx=${idx}" + echo "removing OI files on $facet: idx=${idx}" # step 1: build mount point ${rcmd} mkdir -p $mntpt @@ -6359,41 +6625,28 @@ generate_logname() { echo "$TESTLOG_PREFIX.$TESTNAME.$logname.$(hostname -s).log" } -# mkdir directory on different MDTs +# make directory on different MDTs test_mkdir() { local option local parent local child local path - local dir local rc=0 - if [ $# -eq 2 ]; then - option=$1 - path=$2 - else - path=$1 - fi - - child=${path##*/} - parent=${path%/*} + case $# in + 1) path=$1;; + 2) option=$1 + path=$2;; + *) error "Only creating single directory is supported";; + esac - if [ "$parent" == "$child" ]; then - parent=$(pwd) - fi + child=$(basename $path) + parent=$(dirname $path) - if [ "$option" == "-p" -a -d ${parent}/${child} ]; then + if [ "$option" == "-p" -a -d $parent/$child ]; then return $rc fi - # it needs to check whether there is further / in child - dir=$(echo $child | awk -F '/' '{print $2}') - if [ ! -z "$dir" ]; then - local subparent=$(echo $child | awk -F '/' '{ print $1 }') - parent=${parent}"/"${subparent} - child=$dir - fi - if [ ! -d ${parent} ]; then if [ "$option" == "-p" ]; then mkdir -p ${parent} @@ -6403,19 +6656,30 @@ test_mkdir() { fi if [ $MDSCOUNT -le 1 ]; then - mkdir $option ${parent}/${child} || rc=$? + mkdir $option $parent/$child || rc=$? else local mdt_idx=$($LFS getstripe -M $parent) + local test_num=$(echo $testnum | sed -e 's/[^0-9]*//g') if [ "$mdt_idx" -ne 0 ]; then - mkdir $option ${parent}/${child} || rc=$? - return $rc + mkdir $option $parent/$child || rc=$? + else + mdt_idx=$((test_num % MDSCOUNT)) + echo "mkdir $mdt_idx for $parent/$child" + $LFS setdirstripe -i $mdt_idx $parent/$child || rc=$? fi - - local test_num=$(echo $testnum | sed -e 's/[^0-9]*//g') - local mdt_idx=$((test_num % MDSCOUNT)) - echo "mkdir $mdt_idx for ${parent}/${child}" - $LFS setdirstripe -i $mdt_idx ${parent}/${child} || rc=$? fi return $rc } + +# find the smallest and not in use file descriptor +free_fd() +{ + local max_fd=$(ulimit -n) + local fd=3 + while [[ $fd -le $max_fd && -e /proc/self/fd/$fd ]]; do + ((++fd)) + done + [ $fd -lt $max_fd ] || error "finding free file descriptor failed" + echo $fd +}