X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=9037f7e215423cd21ce57159256dacdc78d917b1;hp=27d06068bf48da5a46b7e92c056bc26aaaab97b7;hb=5764a8ce591e7f62cad2ee95b7041c115a1cd0f6;hpb=af8129c6039ecf3e23b722270ddbb786248a5164 diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 27d0606..9037f7e 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -6,7 +6,6 @@ set -e #set -x export LANG=en_US -export EJOURNAL=${EJOURNAL:-""} export REFORMAT=${REFORMAT:-""} export WRITECONF=${WRITECONF:-""} export VERBOSE=${VERBOSE:-false} @@ -24,6 +23,11 @@ export LOAD_LLOOP=${LOAD_LLOOP:-false} #export PDSH="pdsh -S -Rssh -w" export MOUNT_CMD=${MOUNT_CMD:-"mount -t lustre"} +export UMOUNT=${UMOUNT:-"umount -d"} +# sles12 umount has a issue with -d option +[ -e /etc/SuSE-release ] && grep -w VERSION /etc/SuSE-release | grep -wq 12 && { + export UMOUNT="umount" +} # function used by scripts run on remote nodes LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} @@ -71,7 +75,7 @@ usage() { print_summary () { trap 0 - [ "$TESTSUITE" == "lfsck" ] && return 0 + [ -z "$DEFAULT_SUITES"] && return 0 [ -n "$ONLY" ] && echo "WARNING: ONLY is set to $(echo $ONLY)" local details local form="%-13s %-17s %-9s %s %s\n" @@ -193,9 +197,8 @@ init_test_env() { fi fi - export LFSCK_BIN=${LFSCK_BIN:-lfsck} - export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after each test suite - export FSCK_MAX_ERR=4 # File system errors left uncorrected + export LFSCK_ALWAYS=${LFSCK_ALWAYS:-"no"} # check fs after test suite + export FSCK_MAX_ERR=4 # File system errors left uncorrected export ZFS=${ZFS:-zfs} export ZPOOL=${ZPOOL:-zpool} @@ -229,6 +232,7 @@ init_test_env() { # Ubuntu, at least, has a truncate command in /usr/bin # so fully path our truncate command. export TRUNCATE=${TRUNCATE:-$LUSTRE/tests/truncate} + export FSX=${FSX:-$LUSTRE/tests/fsx} export MDSRATE=${MDSRATE:-"$LUSTRE/tests/mpi/mdsrate"} [ ! -f "$MDSRATE" ] && export MDSRATE=$(which mdsrate 2> /dev/null) if ! echo $PATH | grep -q $LUSTRE/tests/racer; then @@ -267,6 +271,9 @@ init_test_env() { export LFS_MIGRATE=${LFS_MIGRATE:-$LUSTRE/scripts/lfs_migrate} [ ! -f "$LFS_MIGRATE" ] && export LFS_MIGRATE=$(which lfs_migrate 2> /dev/null) + export LR_READER=${LR_READER:-"$LUSTRE/utils/lr_reader"} + [ ! -f "$LR_READER" ] && export LR_READER=$(which lr_reader 2> /dev/null) + [ -z "$LR_READER" ] && export LR_READER="/usr/sbin/lr_reader" export NAME=${NAME:-local} export LGSSD=${LGSSD:-"$LUSTRE/utils/gss/lgssd"} [ "$GSS_PIPEFS" = "true" ] && [ ! -f "$LGSSD" ] && \ @@ -338,6 +345,8 @@ init_test_env() { if ! $RPC_MODE; then rm -f $TMP/*active fi + + export TF_FAIL=${TF_FAIL:-$TMP/tf.fail} } check_cpt_number() { @@ -588,7 +597,7 @@ load_modules_local() { # 'mount' doesn't look in $PATH, just sbin local mount_lustre=$LUSTRE/utils/mount.lustre if [ -f $mount_lustre ]; then - local sbin_mount=/sbin/mount.lustre + local sbin_mount=$(readlink -f /sbin)/mount.lustre if grep -qw "$sbin_mount" /proc/mounts; then cmp -s $mount_lustre $sbin_mount || umount $sbin_mount fi @@ -652,7 +661,7 @@ unload_modules() { fi fi - local sbin_mount=/sbin/mount.lustre + local sbin_mount=$(readlink -f /sbin)/mount.lustre if grep -qe "$sbin_mount " /proc/mounts; then umount $sbin_mount || true [ -s $sbin_mount ] && ! grep -q "STUB MARK" $sbin_mount || @@ -808,7 +817,7 @@ facet_type() { facet_number() { local facet=$1 - if [ $facet == mgs ]; then + if [ $facet == mgs ] || [ $facet == client ]; then return 1 fi @@ -1107,7 +1116,7 @@ get_osd_param() { local name=$3 do_nodes $nodes "$LCTL get_param -n obdfilter.$device.$name \ - osd-*.$device.$name 2>&1" | grep -v 'Found no match' + osd-*.$device.$name 2>&1" | grep -v 'error:' } set_osd_param() { @@ -1117,7 +1126,7 @@ set_osd_param() { local value=$4 do_nodes $nodes "$LCTL set_param -n obdfilter.$device.$name=$value \ - osd-*.$device.$name=$value 2>&1" | grep -v 'Found no match' + osd-*.$device.$name=$value 2>&1" | grep -v 'error:' } set_debug_size () { @@ -1214,6 +1223,10 @@ mount_facet() { local opt=${facet}_opt local mntpt=$(facet_mntpt $facet) local opts="${!opt} $@" + local fstype=$(facet_fstype $facet) + local devicelabel + + module_loaded lustre || load_modules if [ $(facet_fstype $facet) == ldiskfs ] && ! do_facet $facet test -b ${!dev}; then @@ -1225,6 +1238,16 @@ mount_facet() { import_zpool $facet || return ${PIPESTATUS[0]} fi + case $fstype in + ldiskfs) + devicelabel=$(do_facet ${facet} "$E2LABEL ${!dev}");; + zfs) + devicelabel=$(do_facet ${facet} "$ZFS get -H -o value \ + lustre:svname ${!dev}");; + *) + error "unknown fstype!";; + esac + echo "Starting ${facet}: $opts ${!dev} $mntpt" # for testing LU-482 error handling in mount_facets() and test_0a() if [ -f $TMP/test-lu482-trigger ]; then @@ -1234,40 +1257,68 @@ mount_facet() { ${!dev} $mntpt" RC=${PIPESTATUS[0]} fi + if [ $RC -ne 0 ]; then echo "Start of ${!dev} on ${facet} failed ${RC}" - else - set_default_debug_facet $facet + return $RC + fi + + set_default_debug_facet $facet if [[ $facet == mds* ]]; then do_facet $facet \ - lctl set_param -n mdt.${FSNAME}*.enable_remote_dir=1 \ - 2>/dev/null + lctl set_param -n mdt.${FSNAME}*.enable_remote_dir=1 2>/dev/null fi - label=$(devicelabel ${facet} ${!dev}) - [ -z "$label" ] && echo no label for ${!dev} && exit 1 - eval export ${facet}_svc=${label} - echo Started ${label} - fi - return $RC + if [[ $opts =~ .*nosvc.* ]]; then + echo "Start ${!dev} without service" + else + + case $fstype in + ldiskfs) + wait_update_facet ${facet} "$E2LABEL ${!dev} \ + 2>/dev/null | grep -E ':[a-zA-Z]{3}[0-9]{4}'" \ + "" || error "${!dev} failed to initialize!";; + zfs) + wait_update_facet ${facet} "$ZFS get -H -o value \ + lustre:svname ${!dev} 2>/dev/null | \ + grep -E ':[a-zA-Z]{3}[0-9]{4}'" "" || + error "${!dev} failed to initialize!";; + + *) + error "unknown fstype!";; + esac + fi + + # commit the device label change to disk + if [[ $devicelabel =~ (:[a-zA-Z]{3}[0-9]{4}) ]]; then + do_facet $facet "sync; sync; sync" + fi + + + label=$(devicelabel ${facet} ${!dev}) + [ -z "$label" ] && echo no label for ${!dev} && exit 1 + eval export ${facet}_svc=${label} + echo Started ${label} + + return $RC } # start facet device options start() { - local facet=$1 - shift - local device=$1 - shift - eval export ${facet}_dev=${device} - eval export ${facet}_opt=\"$@\" + local facet=$1 + shift + local device=$1 + shift + eval export ${facet}_dev=${device} + eval export ${facet}_opt=\"$@\" - local varname=${facet}failover_dev - if [ -n "${!varname}" ] ; then - eval export ${facet}failover_dev=${!varname} - else - eval export ${facet}failover_dev=$device - fi + local varname=${facet}failover_dev + if [ -n "${!varname}" ] ; then + eval export ${facet}failover_dev=${!varname} + else + eval export ${facet}failover_dev=$device + fi local mntpt=$(facet_mntpt $facet) do_facet ${facet} mkdir -p $mntpt @@ -1295,7 +1346,7 @@ stop() { running=$(do_facet ${facet} "grep -c $mntpt' ' /proc/mounts") || true if [ ${running} -ne 0 ]; then echo "Stopping $mntpt (opts:$@) on $HOST" - do_facet ${facet} umount -d $@ $mntpt + do_facet ${facet} $UMOUNT $@ $mntpt fi # umount should block, but we should wait for unrelated obd's @@ -1348,70 +1399,6 @@ quota_type() { return $rc } -# XXX This function is kept for interoperability with old server (< 2.3.50), -# it should be removed whenever we drop the interoperability for such -# server. -restore_quota_old() { - local mntpt=${1:-$MOUNT} - local quota_type=$(quota_type $FSNAME | grep MDT | cut -d "=" -f2) - if [ ! "$old_QUOTA_TYPE" ] || - [ "$quota_type" = "$old_QUOTA_TYPE" ]; then - return - fi - quota_save_version $old_QUOTA_TYPE -} - -# XXX This function is kept for interoperability with old server (< 2.3.50), -# it should be removed whenever we drop the interoperability for such -# server. -setup_quota_old(){ - local mntpt=$1 - - # no quota enforcement for now and accounting works out of the box - return - - # We need save the original quota_type params, and restore them after testing - - # Suppose that quota type the same on mds and ost - local quota_type=$(quota_type | grep MDT | cut -d "=" -f2) - [ ${PIPESTATUS[0]} -eq 0 ] || error "quota_type failed!" - echo "[HOST:$HOSTNAME] [old_quota_type:$quota_type] [new_quota_type:$QUOTA_TYPE]" - if [ "$quota_type" != "$QUOTA_TYPE" ]; then - export old_QUOTA_TYPE=$quota_type - quota_save_version $QUOTA_TYPE - else - qtype=$(tr -c -d "ug" <<< $QUOTA_TYPE) - $LFS quotacheck -$qtype $mntpt || error "quotacheck has failed for $type" - fi - - local quota_usrs=$QUOTA_USERS - - # get_filesystem_size - local disksz=$(lfs_df $mntpt | grep "summary" | awk '{print $2}') - local blk_soft=$((disksz + 1024)) - local blk_hard=$((blk_soft + blk_soft / 20)) # Go 5% over - - local Inodes=$(lfs_df -i $mntpt | grep "summary" | awk '{print $2}') - local i_soft=$Inodes - local i_hard=$((i_soft + i_soft / 20)) - - echo "Total disk size: $disksz block-softlimit: $blk_soft block-hardlimit: - $blk_hard inode-softlimit: $i_soft inode-hardlimit: $i_hard" - - local cmd - for usr in $quota_usrs; do - echo "Setting up quota on $HOSTNAME:$mntpt for $usr..." - for type in u g; do - cmd="$LFS setquota -$type $usr -b $blk_soft -B $blk_hard -i $i_soft -I $i_hard $mntpt" - echo "+ $cmd" - eval $cmd || error "$cmd FAILED!" - done - # display the quota status - echo "Quota settings for $usr : " - $LFS quota -v -u $usr $mntpt || true - done -} - # get mdt quota type mdt_quota_type() { local varsvc=${SINGLEMDS}_svc @@ -1429,11 +1416,6 @@ ost_quota_type() { # restore old quota type settings restore_quota() { - if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then - restore_quota_old - return - fi - if [ "$old_MDT_QUOTA_TYPE" ]; then do_facet mgs $LCTL conf_param \ $FSNAME.quota.mdt=$old_MDT_QUOTA_TYPE @@ -1469,11 +1451,6 @@ mdt_free_inodes() { } setup_quota(){ - if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then - setup_quota_old $1 - return - fi - local mntpt=$1 # save old quota type & set new quota type @@ -1522,25 +1499,37 @@ setup_quota(){ } zconf_mount() { - local client=$1 - local mnt=$2 - local opts=${3:-$MOUNT_OPTS} - opts=${opts:+-o $opts} - local flags=${4:-$MOUNT_FLAGS} - - local device=$MGSNID:/$FSNAME - if [ -z "$mnt" -o -z "$FSNAME" ]; then - echo Bad zconf mount command: opt=$flags $opts dev=$device mnt=$mnt - exit 1 - fi - - echo "Starting client: $client: $flags $opts $device $mnt" - do_node $client mkdir -p $mnt - do_node $client $MOUNT_CMD $flags $opts $device $mnt || return 1 - - set_default_debug_nodes $client + local client=$1 + local mnt=$2 + local opts=${3:-$MOUNT_OPTS} + opts=${opts:+-o $opts} + local flags=${4:-$MOUNT_FLAGS} + + local device=$MGSNID:/$FSNAME$FILESET + if [ -z "$mnt" -o -z "$FSNAME" ]; then + echo "Bad mount command: opt=$flags $opts dev=$device " \ + "mnt=$mnt" + exit 1 + fi + + echo "Starting client: $client: $flags $opts $device $mnt" + do_node $client mkdir -p $mnt + if [ -n "$FILESET" -a -z "$SKIP_FILESET" ];then + do_node $client $MOUNT_CMD $flags $opts $MGSNID:/$FSNAME \ + $mnt || return 1 + #disable FILESET if not supported + do_nodes $client lctl get_param -n \ + mdc.$FSNAME-MDT0000*.import | grep -q subtree || + device=$MGSNID:/$FSNAME + do_node $client mkdir -p $mnt/$FILESET + do_node $client "! grep -q $mnt' ' /proc/mounts || + umount $mnt" + fi + do_node $client $MOUNT_CMD $flags $opts $device $mnt || return 1 + + set_default_debug_nodes $client - return 0 + return 0 } zconf_umount() { @@ -1630,21 +1619,35 @@ sanity_mount_check () { # mount clients if not mouted zconf_mount_clients() { - local clients=$1 - local mnt=$2 - local opts=${3:-$MOUNT_OPTS} - opts=${opts:+-o $opts} - local flags=${4:-$MOUNT_FLAGS} - - local device=$MGSNID:/$FSNAME - if [ -z "$mnt" -o -z "$FSNAME" ]; then - echo Bad zconf mount command: opt=$flags $opts dev=$device mnt=$mnt - exit 1 - fi - - echo "Starting client $clients: $flags $opts $device $mnt" - - do_nodes $clients " + local clients=$1 + local mnt=$2 + local opts=${3:-$MOUNT_OPTS} + opts=${opts:+-o $opts} + local flags=${4:-$MOUNT_FLAGS} + + local device=$MGSNID:/$FSNAME$FILESET + if [ -z "$mnt" -o -z "$FSNAME" ]; then + echo "Bad conf mount command: opt=$flags $opts dev=$device " \ + "mnt=$mnt" + exit 1 + fi + + echo "Starting client $clients: $flags $opts $device $mnt" + if [ -n "$FILESET" -a ! -n "$SKIP_FILESET" ]; then + do_nodes $clients "! grep -q $mnt' ' /proc/mounts || + umount $mnt" + do_nodes $clients $MOUNT_CMD $flags $opts $MGSNID:/$FSNAME \ + $mnt || return 1 + #disable FILESET if not supported + do_nodes $clients lctl get_param -n \ + mdc.$FSNAME-MDT0000*.import | grep -q subtree || + device=$MGSNID:/$FSNAME + do_nodes $clients mkdir -p $mnt/$FILESET + do_nodes $clients "! grep -q $mnt' ' /proc/mounts || + umount $mnt" + fi + + do_nodes $clients " running=\\\$(mount | grep -c $mnt' '); rc=0; if [ \\\$running -eq 0 ] ; then @@ -1654,12 +1657,12 @@ if [ \\\$running -eq 0 ] ; then fi; exit \\\$rc" || return ${PIPESTATUS[0]} - echo "Started clients $clients: " - do_nodes $clients "mount | grep $mnt' '" + echo "Started clients $clients: " + do_nodes $clients "mount | grep $mnt' '" - set_default_debug_nodes $clients + set_default_debug_nodes $clients - return 0 + return 0 } zconf_umount_clients() { @@ -2101,16 +2104,19 @@ wait_update () { } wait_update_facet() { + local verbose= + [ "$1" = "--verbose" ] && verbose="$1" && shift + local facet=$1 shift - wait_update $(facet_active_host $facet) "$@" + wait_update $verbose $(facet_active_host $facet) "$@" } sync_all_data() { do_nodes $(comma_list $(mdts_nodes)) \ - "lctl set_param -n osd*.*MDT*.force_sync 1" + "lctl set_param -n osd*.*MDT*.force_sync=1" do_nodes $(comma_list $(osts_nodes)) \ - "lctl set_param -n osd*.*OS*.force_sync 1" 2>&1 | + "lctl set_param -n osd*.*OS*.force_sync=1" 2>&1 | grep -v 'Found no match' } @@ -2426,6 +2432,13 @@ affected_facets () { } facet_failover() { + local E2FSCK_ON_MDT0=false + if [ "$1" == "--fsck" ]; then + shift + [ $(facet_fstype $SINGLEMDS) == ldiskfs ] && + E2FSCK_ON_MDT0=true + fi + local facets=$1 local sleep_time=$2 local -a affecteds @@ -2459,6 +2472,9 @@ facet_failover() { shutdown_facet $facet done + $E2FSCK_ON_MDT0 && (run_e2fsck $(facet_active_host $SINGLEMDS) \ + $(mdsdevname 1) "-n" || error "Running e2fsck") + for ((index=0; index<$total; index++)); do facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1) echo reboot facets: ${affecteds[index]} @@ -2928,6 +2944,12 @@ get_env_vars() { fi done + for var in VERBOSE; do + if [ -n "${!var}" ]; then + echo -n " $var=${!var}" + fi + done + if [ -n "$FSTYPE" ]; then echo -n " FSTYPE=$FSTYPE" fi @@ -2981,6 +3003,13 @@ do_nodes() { return ${PIPESTATUS[0]} } +## +# Execute commands on a single service's host +# +# The \a facet (service) may be on a local or remote node, which is +# determined at the time the command is run. +# +# usage: do_facet $facet command [arg ...] do_facet() { local facet=$1 shift @@ -3203,7 +3232,7 @@ unmount_ldiskfs() { local dev=$(facet_device $facet) local mnt=$(facet_mntpt $facet) - do_facet $facet umount -d $mnt + do_facet $facet $UMOUNT $mnt } var_name() { @@ -3314,6 +3343,7 @@ cleanup_echo_devs () { cleanupall() { nfs_client_mode && return + cifs_client_mode && return stopall $* cleanup_echo_devs @@ -3382,9 +3412,11 @@ mkfs_opts() { fs_mkfs_opts+="-O large_xattr" fi - fs_mkfs_opts+=${MDSJOURNALSIZE:+" -J size=$MDSJOURNALSIZE"} - if [ ! -z $EJOURNAL ]; then - fs_mkfs_opts+=${MDSJOURNALSIZE:+" device=$EJOURNAL"} + var=${facet}_JRN + if [ -n "${!var}" ]; then + fs_mkfs_opts+=" -J device=${!var}" + else + fs_mkfs_opts+=${MDSJOURNALSIZE:+" -J size=$MDSJOURNALSIZE"} fi fs_mkfs_opts+=${MDSISIZE:+" -i $MDSISIZE"} fi @@ -3395,7 +3427,12 @@ mkfs_opts() { opts+=${OSSCAPA:+" --param=ost.capa=$OSSCAPA"} if [ $fstype == ldiskfs ]; then - fs_mkfs_opts+=${OSTJOURNALSIZE:+" -J size=$OSTJOURNALSIZE"} + var=${facet}_JRN + if [ -n "${!var}" ]; then + fs_mkfs_opts+=" -J device=${!var}" + else + fs_mkfs_opts+=${OSTJOURNALSIZE:+" -J size=$OSTJOURNALSIZE"} + fi fi fi @@ -3440,39 +3477,63 @@ check_ost_indices() { done } -formatall() { +format_mgs() { local quiet if ! $VERBOSE; then quiet=yes fi + echo "Format mgs: $(mgsdevname)" + reformat_external_journal mgs + add mgs $(mkfs_opts mgs $(mgsdevname)) --reformat \ + $(mgsdevname) $(mgsvdevname) ${quiet:+>/dev/null} || exit 10 +} +format_mdt() { + local num=$1 + local quiet + + if ! $VERBOSE; then + quiet=yes + fi + echo "Format mds$num: $(mdsdevname $num)" + reformat_external_journal mds$num + add mds$num $(mkfs_opts mds$num $(mdsdevname ${num})) \ + --reformat $(mdsdevname $num) $(mdsvdevname $num) \ + ${quiet:+>/dev/null} || exit 10 +} + +format_ost() { + local num=$1 + + if ! $VERBOSE; then + quiet=yes + fi + echo "Format ost$num: $(ostdevname $num)" + reformat_external_journal ost$num + add ost$num $(mkfs_opts ost$num $(ostdevname ${num})) \ + --reformat $(ostdevname $num) $(ostvdevname ${num}) \ + ${quiet:+>/dev/null} || exit 10 +} + +formatall() { stopall # We need ldiskfs here, may as well load them all load_modules [ "$CLIENTONLY" ] && return echo Formatting mgs, mds, osts if ! combined_mgs_mds ; then - echo "Format mgs: $(mgsdevname)" - add mgs $(mkfs_opts mgs $(mgsdevname)) --reformat \ - $(mgsdevname) $(mgsvdevname) ${quiet:+>/dev/null} || - exit 10 + format_mgs fi for num in $(seq $MDSCOUNT); do - echo "Format mds$num: $(mdsdevname $num)" - add mds$num $(mkfs_opts mds$num $(mdsdevname ${num})) \ - --reformat $(mdsdevname $num) $(mdsvdevname $num) \ - ${quiet:+>/dev/null} || exit 10 + format_mdt $num done export OST_INDICES=($(hostlist_expand "$OST_INDEX_LIST")) check_ost_indices for num in $(seq $OSTCOUNT); do - echo "Format ost$num: $(ostdevname $num)" - add ost$num $(mkfs_opts ost$num $(ostdevname ${num})) \ - --reformat $(ostdevname $num) $(ostvdevname ${num}) \ - ${quiet:+>/dev/null} || exit 10 + format_ost $num done } @@ -3550,6 +3611,7 @@ writeconf_all () { setupall() { nfs_client_mode && return + cifs_client_mode && return sanity_mount_check || error "environments are insane!" @@ -3815,6 +3877,11 @@ nfs_client_mode () { return 1 } +cifs_client_mode () { + [ x$CIFSCLIENT = xyes ] && + echo "CIFSCLIENT=$CIFSCLIENT mode: setup, cleanup, check config skipped" +} + check_config_client () { local mntpt=$1 @@ -3862,6 +3929,7 @@ check_config_clients () { local mntpt=$1 nfs_client_mode && return + cifs_client_mode && return do_rpc_nodes "$clients" check_config_client $mntpt @@ -3908,6 +3976,7 @@ is_empty_fs() { check_and_setup_lustre() { nfs_client_mode && return + cifs_client_mode && return local MOUNTED=$(mounted_lustre_filesystems) @@ -4120,88 +4189,31 @@ check_shared_dir() { return 0 } -# Run e2fsck on MDT and OST(s) to generate databases used for lfsck. -generate_db() { - local i - local ostidx - local dev - local node - - [[ $(lustre_version_code $SINGLEMDS) -ne $(version_code 2.2.0) ]] || - { skip "Lustre 2.2.0 lacks the patch for LU-1255"; exit 0; } - - check_shared_dir $SHARED_DIRECTORY || - error "$SHARED_DIRECTORY isn't a shared directory" - - export MDSDB=$SHARED_DIRECTORY/mdsdb - export OSTDB=$SHARED_DIRECTORY/ostdb - - # DNE is not supported, so when running e2fsck on a DNE filesystem, - # we only pass master MDS parameters. - run_e2fsck $MDTNODE $MDTDEV "-n --mdsdb $MDSDB" - - i=0 - ostidx=0 - OSTDB_LIST="" - for node in $(osts_nodes); do - for dev in ${OSTDEVS[i]}; do - run_e2fsck $node $dev "-n --mdsdb $MDSDB --ostdb $OSTDB-$ostidx" - OSTDB_LIST="$OSTDB_LIST $OSTDB-$ostidx" - ostidx=$((ostidx + 1)) - done - i=$((i + 1)) - done -} - -# Run lfsck on server node if lfsck can't be found on client (LU-2571) -run_lfsck_remote() { - local cmd="$LFSCK_BIN -c -l --mdsdb $MDSDB --ostdb $OSTDB_LIST $MOUNT" - local client=$1 - local mounted=true - local rc=0 - - #Check if lustre is already mounted - do_rpc_nodes $client is_mounted $MOUNT || mounted=false - if ! $mounted; then - zconf_mount $client $MOUNT || - error "failed to mount Lustre on $client" - fi - #Run lfsck - echo $cmd - do_node $client $cmd || rc=$? - #Umount if necessary - if ! $mounted; then - zconf_umount $client $MOUNT || - error "failed to unmount Lustre on $client" - fi - - [ $rc -le $FSCK_MAX_ERR ] || - error "$cmd returned $rc, should be <= $FSCK_MAX_ERR" - echo "lfsck finished with rc=$rc" - - return $rc -} - run_lfsck() { - local facets="client $SINGLEMDS" - local found=false - local facet - local node - local rc=0 - - for facet in $facets; do - node=$(facet_active_host $facet) - if check_progs_installed $node $LFSCK_BIN; then - found=true - break - fi + do_nodes $(comma_list $(mdts_nodes) $(osts_nodes)) \ + $LCTL set_param printk=+lfsck + do_facet $SINGLEMDS "$LCTL lfsck_start -M $FSNAME-MDT0000 -r -A -t all" + + for k in $(seq $MDSCOUNT); do + # wait up to 10+1 minutes for LFSCK to complete + wait_update_facet --verbose mds${k} "$LCTL get_param -n \ + mdd.$(facet_svc mds${k}).lfsck_layout | + awk '/^status/ { print \\\$2 }'" "completed" 600 || + error "MDS${k} layout isn't the expected 'completed'" + wait_update_facet --verbose mds${k} "$LCTL get_param -n \ + mdd.$(facet_svc mds${k}).lfsck_namespace | + awk '/^status/ { print \\\$2 }'" "completed" 60 || + error "MDS${k} namespace isn't the expected 'completed'" done - ! $found && error "None of \"$facets\" supports lfsck" - - run_lfsck_remote $node || rc=$? - - rm -rvf $MDSDB* $OSTDB* || true - return $rc + local rep_mdt=$(do_nodes $(comma_list $(mdts_nodes)) \ + $LCTL get_param -n mdd.$FSNAME-*.lfsck_* | + awk '/repaired/ { print $2 }' | calc_sum) + local rep_ost=$(do_nodes $(comma_list $(osts_nodes)) \ + $LCTL get_param -n obdfilter.$FSNAME-*.lfsck_* | + awk '/repaired/ { print $2 }' | calc_sum) + local repaired=$((rep_mdt + rep_ost)) + [ $repaired -eq 0 ] || + error "lfsck repaired $rep_mdt MDT and $rep_ost OST errors" } dump_file_contents() { @@ -4259,11 +4271,10 @@ log_zfs_info() { } check_and_cleanup_lustre() { - if [ "$LFSCK_ALWAYS" = "yes" -a "$TESTSUITE" != "lfsck" ]; then - get_svr_devs - generate_db - run_lfsck - fi + if [ "$LFSCK_ALWAYS" = "yes" -a "$TESTSUITE" != "sanity-lfsck" -a \ + "$TESTSUITE" != "sanity-scrub" ]; then + run_lfsck + fi if is_mounted $MOUNT; then [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* || @@ -4594,33 +4605,10 @@ set_nodes_failloc () { } cancel_lru_locks() { - $LCTL mark "cancel_lru_locks $1 start" - - if [ $1 != "MGC" ]; then - for d in $(lctl get_param -N ldlm.namespaces.*.lru_size | - egrep -i $1); do - $LCTL set_param -n $d=clear - done - $LCTL get_param ldlm.namespaces.*.lock_unused_count | egrep -i $1 | - grep -v '=0' - else - for d in $(find \ - /{proc,sys}/fs/lustre/ldlm/namespaces/*$1*/lru_size \ - 2> /dev/null); do - echo "clear" > $d - done - - for d in $(find \ - /{proc,sys}/fs/lustre/ldlm/namespaces/*$1*/lock_unused_count \ - 2> /dev/null); do - if [ $(cat $d) != 0 ]; then - echo "ldlm.namespaces.$(echo "$d" | - cut -f 7 -d'/').lock_unused_count=$(cat $d)" - fi - done - fi - - $LCTL mark "cancel_lru_locks $1 stop" + #$LCTL mark "cancel_lru_locks $1 start" + $LCTL set_param -n ldlm.namespaces.*$1*.lru_size=clear + $LCTL get_param ldlm.namespaces.*$1*.lock_unused_count | grep -v '=0' + #$LCTL mark "cancel_lru_locks $1 stop" } default_lru_size() @@ -4742,6 +4730,9 @@ error_noexit() { echo "$@" > $LOGDIR/err fi fi + + # cleanup the env for failed tests + reset_fail_loc } exit_status () { @@ -4984,6 +4975,18 @@ banner() { log "$msg== $(date +"%H:%M:%S (%s)")" } +check_dmesg_for_errors() { + local res + local errors="VFS: Busy inodes after unmount of\|\ +ldiskfs_check_descriptors: Checksum for group 0 failed\|\ +group descriptors corrupted" + + res=$(do_nodes $(comma_list $(nodes_list)) "dmesg" | grep "$errors") + [ -z "$res" ] && return 0 + echo "Kernel error detected: $res" + return 1 +} + # # Run a single test function and cleanup after it. # @@ -4999,12 +5002,17 @@ run_one() { local SAVE_UMASK=`umask` umask 0022 + if ! grep -q $DIR /proc/mounts; then + $SETUP + fi + banner "test $testnum: $message" test_${testnum} || error "test_$testnum failed with $?" cd $SAVE_PWD reset_fail_loc check_grant ${testnum} || error "check_grant $testnum failed with $?" check_catastrophe || error "LBUG/LASSERT detected" + check_dmesg_for_errors || error "Error in dmesg detected" if [ "$PARALLEL" != "yes" ]; then ps auxww | grep -v grep | grep -q multiop && error "multiop still running" @@ -5013,6 +5021,7 @@ run_one() { unset tdir unset tfile umask $SAVE_UMASK + $CLEANUP return 0 } @@ -5240,6 +5249,7 @@ require_dsh_ost() remote_mgs_nodsh() { + [ "$CLIENTONLY" ] && return 0 || true local MGS MGS=$(facet_host mgs) remote_node $MGS && [ "$PDSH" = "no_dsh" -o -z "$PDSH" -o -z "$ost_HOST" ] @@ -5764,19 +5774,21 @@ mds_on_old_device() { } get_mdtosc_proc_path() { - local mds_facet=$1 - local ost_label=${2:-"*OST*"} - - [ "$mds_facet" = "mds" ] && mds_facet=$SINGLEMDS - local mdt_label=$(convert_facet2label $mds_facet) - local mdt_index=$(echo $mdt_label | sed -e 's/^.*-//') - - if [ $(lustre_version_code $mds_facet) -le $(version_code 1.8.0) ] || - mds_on_old_device $mds_facet; then - echo "${ost_label}-osc" - else - echo "${ost_label}-osc-${mdt_index}" - fi + local mds_facet=$1 + local ost_label=${2:-"*OST*"} + + [ "$mds_facet" = "mds" ] && mds_facet=$SINGLEMDS + local mdt_label=$(convert_facet2label $mds_facet) + local mdt_index=$(echo $mdt_label | sed -e 's/^.*-//') + + if [ $(lustre_version_code $mds_facet) -le $(version_code 1.8.0) ] || + mds_on_old_device $mds_facet; then + echo "${ost_label}-osc" + elif [[ $ost_label = *OST* ]]; then + echo "${ost_label}-osc-${mdt_index}" + else + echo "${ost_label}-osp-${mdt_index}" + fi } get_osc_import_name() { @@ -5797,6 +5809,7 @@ _wait_import_state () { local expected=$1 local CONN_PROC=$2 local maxtime=${3:-$(max_recovery_time)} + local error_on_failure=${4:-1} local CONN_STATE local i=0 @@ -5809,9 +5822,12 @@ _wait_import_state () { # reconnect timeout and test can't see real disconnect [ "${CONN_STATE}" == "CONNECTING" ] && return 0 fi - [ $i -ge $maxtime ] && \ - error "can't put import for $CONN_PROC into ${expected} state after $i sec, have ${CONN_STATE}" && \ + if [ $i -ge $maxtime ]; then + [ $error_on_failure -ne 0 ] && \ + error "can't put import for $CONN_PROC into ${expected}" \ + "state after $i sec, have ${CONN_STATE}" return 1 + fi sleep 1 # Add uniq for multi-mount case CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2 | uniq) @@ -5826,10 +5842,11 @@ wait_import_state() { local state=$1 local params=$2 local maxtime=${3:-$(max_recovery_time)} + local error_on_failure=${4:-1} local param for param in ${params//,/ }; do - _wait_import_state $state $param $maxtime || return + _wait_import_state $state $param $maxtime $error_on_failure || return done } @@ -5885,8 +5902,8 @@ _wait_osc_import_state() { local facet=$1 local ost_facet=$2 local expected=$3 - local ost=$(get_osc_import_name $facet $ost_facet) - local param="osc.${ost}.ost_server_uuid" + local target=$(get_osc_import_name $facet $ost_facet) + local param="osc.${target}.ost_server_uuid" local params=$param local i=0 @@ -5910,6 +5927,16 @@ _wait_osc_import_state() { params=$($LCTL list_param $param 2>/dev/null || true) done fi + + if [[ $ost_facet = mds* ]]; then + # no OSP connection to itself + if [[ $facet = $ost_facet ]]; then + return 0 + fi + param="osp.${target}.mdt_server_uuid" + params=$param + fi + if ! do_rpc_nodes "$(facet_active_host $facet)" \ wait_import_state $expected "$params" $maxtime; then error "import is not in ${expected} state" @@ -5934,6 +5961,73 @@ wait_osc_import_state() { fi } +_wait_mgc_import_state() { + local facet=$1 + local expected=$2 + local error_on_failure=${3:-1} + local param="mgc.*.mgs_server_uuid" + local params=$param + local i=0 + + # 1. wait the deadline of client 1st request (it could be skipped) + # 2. wait the deadline of client 2nd request + local maxtime=$(( 2 * $(request_timeout $facet))) + + if [[ $facet == client* ]]; then + # During setup time, the osc might not be setup, it need wait + # until list_param can return valid value. And also if there + # are mulitple osc entries we should list all of them before + # go to wait. + params=$($LCTL list_param $param 2>/dev/null || true) + while [ -z "$params" ]; do + if [ $i -ge $maxtime ]; then + echo "can't get $param in $maxtime secs" + return 1 + fi + sleep 1 + i=$((i + 1)) + params=$($LCTL list_param $param 2>/dev/null || true) + done + fi + if ! do_rpc_nodes "$(facet_active_host $facet)" \ + wait_import_state $expected "$params" $maxtime \ + $error_on_failure; then + if [ $error_on_failure -ne 0 ]; then + error "import is not in ${expected} state" + fi + return 1 + fi + + return 0 +} + +wait_mgc_import_state() { + local facet=$1 + local expected=$2 + local error_on_failure=${3:-1} + local num + + if [[ $facet = mds ]]; then + for num in $(seq $MDSCOUNT); do + _wait_mgc_import_state mds$num "$expected" \ + $error_on_failure || return + done + else + _wait_mgc_import_state "$facet" "$expected" + $error_on_failure || return + fi +} + +wait_dne_interconnect() { + local num + + if [ $MDSCOUNT -gt 1 ]; then + for num in $(seq $MDSCOUNT); do + wait_osc_import_state mds mds$num FULL + done + fi +} + get_clientmdc_proc_path() { echo "${1}-mdc-*" } @@ -6040,19 +6134,21 @@ create_pool() { } add_pool_to_list () { - local fsname=${1%%.*} - local poolname=${1##$fsname.} + local fsname=${1%%.*} + local poolname=${1##$fsname.} - local listvar=${fsname}_CREATED_POOLS - eval export ${listvar}=$(expand_list ${!listvar} $poolname) + local listvar=${fsname}_CREATED_POOLS + local temp=${listvar}=$(expand_list ${!listvar} $poolname) + eval export $temp } remove_pool_from_list () { - local fsname=${1%%.*} - local poolname=${1##$fsname.} + local fsname=${1%%.*} + local poolname=${1##$fsname.} - local listvar=${fsname}_CREATED_POOLS - eval export ${listvar}=$(exclude_items_from_list ${!listvar} $poolname) + local listvar=${fsname}_CREATED_POOLS + local temp=${listvar}=$(exclude_items_from_list ${!listvar} $poolname) + eval export $temp } destroy_pool_int() { @@ -6189,6 +6285,31 @@ max_recovery_time() { echo -n $service_time } +recovery_time_min() { + local connection_switch_min=5 + local connection_switch_inc=5 + local connection_switch_max + local reconnect_delay_max + local initial_connect_timeout + local max + local timout_20 + + #connection_switch_max=min(50, max($connection_switch_min,$TIMEOUT) + (($connection_switch_min > $TIMEOUT)) && + max=$connection_switch_min || max=$TIMEOUT + (($max < 50)) && connection_switch_max=$max || connection_switch_max=50 + + #initial_connect_timeout = max(connection_switch_min, obd_timeout/20) + timeout_20=$((TIMEOUT/20)) + (($connection_switch_min > $timeout_20)) && + initial_connect_timeout=$connection_switch_min || + initial_connect_timeout=$timeout_20 + + reconnect_delay_max=$((connection_switch_max + connection_switch_inc + \ + initial_connect_timeout)) + echo $((2 * reconnect_delay_max)) +} + get_clients_mount_count () { local clients=${CLIENTS:-`hostname`} @@ -6815,12 +6936,14 @@ generate_string() { reformat_external_journal() { local facet=$1 + local var - if [ ! -z ${EJOURNAL} ]; then + var=${facet}_JRN + if [ -n "${!var}" ]; then local rcmd="do_facet $facet" - echo "reformat external journal on $facet:${EJOURNAL}" - ${rcmd} mke2fs -O journal_dev ${EJOURNAL} || return 1 + echo "reformat external journal on $facet:${!var}" + ${rcmd} mke2fs -O journal_dev ${!var} || return 1 fi } @@ -6860,14 +6983,10 @@ mds_backup_restore() { echo "backup data" ${rcmd} tar zcf $metadata -C $mntpt/ . > /dev/null 2>&1 || return 3 # step 6: umount - ${rcmd} umount -d $mntpt || return 4 - # step 7: reformat external journal if needed - reformat_external_journal $facet || return 5 + ${rcmd} $UMOUNT $mntpt || return 4 # step 8: reformat dev echo "reformat new device" - add $facet $(mkfs_opts $facet ${devname}) --backfstype ldiskfs \ - --reformat ${devname} $(mdsvdevname $(facet_number $facet)) \ - > /dev/null || exit 6 + format_mdt $(facet_number $facet) # step 9: mount dev ${rcmd} mount -t ldiskfs $opts $devname $mntpt || return 7 # step 10: restore metadata @@ -6880,7 +6999,7 @@ mds_backup_restore() { echo "remove recovery logs" ${rcmd} rm -fv $mntpt/OBJECTS/* $mntpt/CATALOGS # step 13: umount dev - ${rcmd} umount -d $mntpt || return 10 + ${rcmd} $UMOUNT $mntpt || return 10 # step 14: cleanup tmp backup ${rcmd} rm -f $metaea $metadata # step 15: reset device label - it's not virgin on @@ -6920,7 +7039,7 @@ mds_remove_ois() { done fi # step 4: umount - ${rcmd} umount -d $mntpt || return 2 + ${rcmd} $UMOUNT $mntpt || return 2 # OI files will be recreated when mounted as lustre next time. } @@ -7041,7 +7160,7 @@ pool_add() { create_pool $FSNAME.$pool || { error_noexit "No pool created, result code $?"; return 1; } - [ $($LFS pool_list $FSNAME | grep -c $pool) -eq 1 ] || + [ $($LFS pool_list $FSNAME | grep -c "$FSNAME.${pool}\$") -eq 1 ] || { error_noexit "$pool not in lfs pool_list"; return 2; } } @@ -7316,3 +7435,12 @@ check_start_ost_idx() { error "OST index of the first stripe on $file is" \ "$start_ost_idx, should be $expected" } + +killall_process () { + local clients=${1:-$(hostname)} + local name=$2 + local signal=$3 + local rc=0 + + do_nodes $clients "killall $signal $name" +}