X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=47dfe99d68f3f016a7998d76f38b1ce7761e3942;hp=25c66698b0e29f4da6e566754112a351c3cdc683;hb=c0d3e6a3b639104a262b34be3e5753dd8f5dcdac;hpb=691fffacfd3b9e9020c615281b409b34eef2dd90 diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 25c6669..47dfe99 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -5,6 +5,7 @@ trap 'print_summary && touch $TF_FAIL && \ set -e #set -x +export LANG=en_US export EJOURNAL=${EJOURNAL:-""} export REFORMAT=${REFORMAT:-""} export WRITECONF=${WRITECONF:-""} @@ -15,8 +16,8 @@ export GSS_KRB5=false export GSS_PIPEFS=false export IDENTITY_UPCALL=default export QUOTA_AUTO=1 -export JOBSTATS_AUTO=${JOBSTATS_AUTO:-1} -export JOBID_VAR=${JOBID_VAR:-"procname_uid"} +# specify environment variable containing batch job name for server statistics +export JOBID_VAR=${JOBID_VAR:-"procname_uid"} # or "existing" or "disable" # LOAD_LLOOP: LU-409: only load llite_lloop module if kernel < 2.6.32 or # LOAD_LLOOP is true. LOAD_LLOOP is false by default. @@ -185,6 +186,7 @@ init_test_env() { export ZFS=${ZFS:-zfs} export ZPOOL=${ZPOOL:-zpool} export ZDB=${ZDB:-zdb} + export PARTPROBE=${PARTPROBE:-partprobe} #[ -d /r ] && export ROOT=${ROOT:-/r} export TMP=${TMP:-$ROOT/tmp} @@ -290,13 +292,14 @@ init_test_env() { export RLUSTRE=${RLUSTRE:-$LUSTRE} export RPWD=${RPWD:-$PWD} export I_MOUNTED=${I_MOUNTED:-"no"} - if [ ! -f /lib/modules/$(uname -r)/kernel/fs/lustre/mdt.ko -a \ - ! -f /lib/modules/$(uname -r)/updates/kernel/fs/lustre/mdt.ko -a \ - ! -f `dirname $0`/../mdt/mdt.ko ]; then - export CLIENTMODSONLY=yes - fi + if [ ! -f /lib/modules/$(uname -r)/kernel/fs/lustre/mdt.ko -a \ + ! -f /lib/modules/$(uname -r)/updates/kernel/fs/lustre/mdt.ko -a \ + ! -f $LUSTRE/mdt/mdt.ko ]; then + export CLIENTMODSONLY=yes + fi - export SHUTDOWN_ATTEMPTS=${SHUTDOWN_ATTEMPTS:-3} + export SHUTDOWN_ATTEMPTS=${SHUTDOWN_ATTEMPTS:-3} + export OSD_TRACK_DECLARES_LBUG=${OSD_TRACK_DECLARES_LBUG:-"yes"} # command line @@ -473,7 +476,6 @@ load_modules_local() { load_module ../lnet/lnet/lnet LNETLND=${LNETLND:-"socklnd/ksocklnd"} load_module ../lnet/klnds/$LNETLND - load_module lvfs/lvfs load_module obdclass/obdclass load_module ptlrpc/ptlrpc load_module ptlrpc/gss/ptlrpc_gss @@ -490,24 +492,20 @@ load_modules_local() { grep -q crc16 $SYMLIST || { modprobe crc16 2>/dev/null || true; } grep -q -w jbd $SYMLIST || { modprobe jbd 2>/dev/null || true; } grep -q -w jbd2 $SYMLIST || { modprobe jbd2 2>/dev/null || true; } + load_module lfsck/lfsck [ "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS if [[ $(node_fstypes $HOSTNAME) == *zfs* ]]; then modprobe zfs load_module osd-zfs/osd_zfs fi - load_module mgs/mgs - load_module mdd/mdd if [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then - # - # This block shall be moved up beside osd-zfs as soon - # as osd-ldiskfs stops using mdd symbols. - # grep -q exportfs_decode_fh $SYMLIST || { modprobe exportfs 2> /dev/null || true; } - load_module ../ldiskfs/ldiskfs/ldiskfs - load_module lvfs/fsfilt_ldiskfs + load_module ../ldiskfs/ldiskfs load_module osd-ldiskfs/osd_ldiskfs fi + load_module mgs/mgs + load_module mdd/mdd load_module mdt/mdt load_module ost/ost load_module lod/lod @@ -591,7 +589,7 @@ fs_log_size() { local size=0 case $fstype in ldiskfs) size=50;; # largest seen is 44, leave some headroom - zfs) size=256;; + zfs) size=400;; # largest seen is 384 esac echo -n $size @@ -709,6 +707,13 @@ cleanup_gss() { fi } +facet_svc() { + local facet=$1 + local var=${facet}_svc + + echo -n ${!var} +} + facet_type() { local facet=$1 @@ -808,6 +813,140 @@ ostdevlabel() { } # +# Get the device of a facet. +# +facet_device() { + local facet=$1 + local device + + case $facet in + mgs) device=$(mgsdevname) ;; + mds*) device=$(mdsdevname $(facet_number $facet)) ;; + ost*) device=$(ostdevname $(facet_number $facet)) ;; + fs2mds) device=$(mdsdevname 1_2) ;; + fs2ost) device=$(ostdevname 1_2) ;; + fs3ost) device=$(ostdevname 2_2) ;; + *) ;; + esac + + echo -n $device +} + +# +# Get the virtual device of a facet. +# +facet_vdevice() { + local facet=$1 + local device + + case $facet in + mgs) device=$(mgsvdevname) ;; + mds*) device=$(mdsvdevname $(facet_number $facet)) ;; + ost*) device=$(ostvdevname $(facet_number $facet)) ;; + fs2mds) device=$(mdsvdevname 1_2) ;; + fs2ost) device=$(ostvdevname 1_2) ;; + fs3ost) device=$(ostvdevname 2_2) ;; + *) ;; + esac + + echo -n $device +} + +# +# Re-read the partition table on failover partner host. +# After a ZFS storage pool is created on a shared device, the partition table +# on the device may change. However, the operating system on the failover +# host may not notice the change automatically. Without the up-to-date partition +# block devices, 'zpool import ..' cannot find the labels, whose positions are +# relative to partition rather than disk beginnings. +# +# This function performs partprobe on the failover host to make it re-read the +# partition table. +# +refresh_partition_table() { + local facet=$1 + local device=$2 + local host + + host=$(facet_passive_host $facet) + if [[ -n "$host" ]]; then + do_node $host "$PARTPROBE $device" + fi +} + +# +# Get ZFS storage pool name. +# +zpool_name() { + local facet=$1 + local device + local poolname + + device=$(facet_device $facet) + # poolname is string before "/" + poolname="${device%%/*}" + + echo -n $poolname +} + +# +# Export ZFS storage pool. +# Before exporting the pool, all datasets within the pool should be unmounted. +# +export_zpool() { + local facet=$1 + shift + local opts="$@" + local poolname + + poolname=$(zpool_name $facet) + + if [[ -n "$poolname" ]]; then + do_facet $facet "! $ZPOOL list -H $poolname >/dev/null 2>&1 || + grep -q ^$poolname/ /proc/mounts || + $ZPOOL export $opts $poolname" + fi +} + +# +# Import ZFS storage pool. +# Force importing, even if the pool appears to be potentially active. +# +import_zpool() { + local facet=$1 + shift + local opts=${@:-"-o cachefile=none"} + local poolname + + poolname=$(zpool_name $facet) + + if [[ -n "$poolname" ]]; then + opts+=" -d $(dirname $(facet_vdevice $facet))" + do_facet $facet "$ZPOOL list -H $poolname >/dev/null 2>&1 || + $ZPOOL import -f $opts $poolname" + fi +} + +# +# Set the "cachefile=none" property on ZFS storage pool so that the pool +# is not automatically imported on system startup. +# +# In a failover environment, this will provide resource level fencing which +# will ensure that the same ZFS storage pool will not be imported concurrently +# on different nodes. +# +disable_zpool_cache() { + local facet=$1 + local poolname + + poolname=$(zpool_name $facet) + + if [[ -n "$poolname" ]]; then + do_facet $facet "$ZPOOL set cachefile=none $poolname" + fi +} + +# # This and set_osd_param() shall be used to access OSD parameters # once existed under "obdfilter": # @@ -935,6 +1074,11 @@ mount_facet() { opts=$(csa_add "$opts" -o loop) fi + if [[ $(facet_fstype $facet) == zfs ]]; then + # import ZFS storage pool + import_zpool $facet || return ${PIPESTATUS[0]} + fi + echo "Starting ${facet}: $opts ${!dev} $mntpt" # for testing LU-482 error handling in mount_facets() and test_0a() if [ -f $TMP/test-lu482-trigger ]; then @@ -981,39 +1125,6 @@ start() { return $RC } -# -# When a ZFS OSD is made read-only by replay_barrier(), its pool is "freezed". -# Because stopping corresponding target may not clear this in-memory state, we -# need to zap the pool from memory by exporting and reimporting the pool. -# -# Although the uberblocks are not updated when a pool is freezed, transactions -# are still written to the disks. Modified blocks may be cached in memory when -# tests try reading them back. The export-and-reimport process also evicts any -# cached pool data from memory to provide the correct "data loss" semantics. -# -refresh_disk() { - local facet=$1 - local fstype=$(facet_fstype $facet) - local _dev - local dev - local poolname - - if [ "${fstype}" == "zfs" ]; then - _dev=$(facet_active $facet)_dev - dev=${!_dev} # expand _dev to its value, e.g. ${mds1_dev} - poolname="${dev%%/*}" # poolname is string before "/" - - if [ "${poolname}" == "" ]; then - echo "invalid dataset name: $dev" - return - fi - do_facet $facet "cp /etc/zfs/zpool.cache /tmp/zpool.cache.back" - do_facet $facet "$ZPOOL export ${poolname}" - do_facet $facet "$ZPOOL import -f -c /tmp/zpool.cache.back \ - ${poolname}" - fi -} - stop() { local running local facet=$1 @@ -1028,9 +1139,14 @@ stop() { do_facet ${facet} umount -d $@ $mntpt fi - # umount should block, but we should wait for unrelated obd's - # like the MGS or MGC to also stop. - wait_exit_ST ${facet} + # umount should block, but we should wait for unrelated obd's + # like the MGS or MGC to also stop. + wait_exit_ST ${facet} || return ${PIPESTATUS[0]} + + if [[ $(facet_fstype $facet) == zfs ]]; then + # export ZFS storage pool + export_zpool $facet + fi } # save quota version (both administrative and operational quotas) @@ -1176,6 +1292,23 @@ lfs_df() { $LFS df $* | sed -e 's/filesystem /filesystem_/' } +# Get free inodes on the MDT specified by mdt index, free indoes on +# the whole filesystem will be returned when index == -1. +mdt_free_inodes() { + local index=$1 + local free_inodes + local mdt_uuid + + if [ $index -eq -1 ]; then + mdt_uuid="summary" + else + mdt_uuid=$(mdtuuid_from_index $index) + fi + + free_inodes=$(lfs_df -i $MOUNT | grep $mdt_uuid | awk '{print $4}') + echo $free_inodes +} + setup_quota(){ if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then setup_quota_old $1 @@ -1488,7 +1621,6 @@ reboot_facet() { if [ "$FAILURE_MODE" = HARD ]; then reboot_node $(facet_active_host $facet) else - refresh_disk ${facet} sleep 10 fi } @@ -1551,6 +1683,7 @@ TESTLOG_PREFIX=$TESTLOG_PREFIX \ TESTNAME=$TESTNAME \ DBENCH_LIB=$DBENCH_LIB \ DBENCH_SRC=$DBENCH_SRC \ +CLIENT_COUNT=$((CLIENTCOUNT - 1)) \ LFS=$LFS \ run_${load}.sh" & local ppid=$! @@ -1760,30 +1893,43 @@ cleanup_check() { } wait_update () { - local node=$1 - local TEST=$2 - local FINAL=$3 - local MAX=${4:-90} - - local RESULT - local WAIT=0 - local sleep=1 - local print=10 - while [ true ]; do - RESULT=$(do_node $node "$TEST") - if [ "$RESULT" == "$FINAL" ]; then - [ -z "$RESULT" -o $WAIT -le $sleep ] || - echo "Updated after ${WAIT}s: wanted '$FINAL' got '$RESULT'" - return 0 - fi - [ $WAIT -ge $MAX ] && break - [ $((WAIT % print)) -eq 0 ] && - echo "Waiting $((MAX - WAIT)) secs for update" - WAIT=$((WAIT + sleep)) - sleep $sleep - done - echo "Update not seen after ${MAX}s: wanted '$FINAL' got '$RESULT'" - return 3 + local verbose=false + if [[ "$1" == "--verbose" ]]; then + shift + verbose=true + fi + + local node=$1 + local TEST=$2 + local FINAL=$3 + local MAX=${4:-90} + local RESULT + local PREV_RESULT + local WAIT=0 + local sleep=1 + local print=10 + + while [ true ]; do + RESULT=$(do_node $node "$TEST") + if [[ "$RESULT" == "$FINAL" ]]; then + [[ -z "$RESULT" || $WAIT -le $sleep ]] || + echo "Updated after ${WAIT}s: wanted '$FINAL'"\ + "got '$RESULT'" + return 0 + fi + if [[ $verbose && "$RESULT" != "$PREV_RESULT" ]]; then + echo "Changed after ${WAIT}s: from '$PREV_RESULT'"\ + "to '$RESULT'" + PREV_RESULT=$RESULT + fi + [[ $WAIT -ge $MAX ]] && break + [[ $((WAIT % print)) -eq 0 ]] && + echo "Waiting $((MAX - WAIT)) secs for update" + WAIT=$((WAIT + sleep)) + sleep $sleep + done + echo "Update not seen after ${MAX}s: wanted '$FINAL' got '$RESULT'" + return 3 } wait_update_facet() { @@ -1793,7 +1939,10 @@ wait_update_facet() { } sync_all_data() { - do_node $(osts_nodes) "lctl set_param -n osd*.*OS*.force_sync 1" 2>&1 | + do_nodes $(comma_list $(mdts_nodes)) \ + "lctl set_param -n osd*.*MDT*.force_sync 1" + do_nodes $(comma_list $(osts_nodes)) \ + "lctl set_param -n osd*.*OS*.force_sync 1" 2>&1 | grep -v 'Found no match' } @@ -1914,6 +2063,7 @@ wait_mds_ost_sync () { echo "Waiting for orphan cleanup..." # MAX value includes time needed for MDS-OST reconnection local MAX=$(( TIMEOUT * 2 )) + local WAIT_TIMEOUT=${1:-$MAX} local WAIT=0 local new_wait=true local list=$(comma_list $(mdts_nodes)) @@ -1926,7 +2076,9 @@ wait_mds_ost_sync () { list=$(comma_list $(osts_nodes)) cmd="$LCTL get_param -n obdfilter.*.mds_sync" fi - while [ $WAIT -lt $MAX ]; do + + echo "wait $WAIT_TIMEOUT secs maximumly for $list mds-ost sync done." + while [ $WAIT -lt $WAIT_TIMEOUT ]; do local -a sync=($(do_nodes $list "$cmd")) local con=1 local i @@ -1942,10 +2094,13 @@ wait_mds_ost_sync () { done sleep 2 # increase waiting time and cover statfs cache [ ${con} -eq 1 ] && return 0 - echo "Waiting $WAIT secs for $facet mds-ost sync done." + echo "Waiting $WAIT secs for $list $i mds-ost sync done." WAIT=$((WAIT + 2)) done - echo "$facet recovery not done in $MAX sec. $STATUS" + + # show which nodes are not finished. + do_nodes $list "$cmd" + echo "$facet recovery node $i not done in $WAIT_TIMEOUT sec. $STATUS" return 1 } @@ -2148,7 +2303,7 @@ replay_barrier() { do_facet $facet "sync; sync; sync" df $MOUNT - # make sure there will be no seq change + # make sure there will be no seq change local clients=${CLIENTS:-$HOSTNAME} local f=fsa-\\\$\(hostname\) do_nodes $clients "mcreate $MOUNT/$f; rm $MOUNT/$f" @@ -2156,6 +2311,21 @@ replay_barrier() { local svc=${facet}_svc do_facet $facet $LCTL --device ${!svc} notransno + # + # If a ZFS OSD is made read-only here, its pool is "freezed". This + # in-memory state has to be cleared by either rebooting the host or + # exporting and reimporting the pool. + # + # Although the uberblocks are not updated when a pool is freezed, + # transactions are still written to the disks. Modified blocks may be + # cached in memory when tests try reading them back. The + # export-and-reimport process also evicts any cached pool data from + # memory to provide the correct "data loss" semantics. + # + # In the test framework, the exporting and importing operations are + # handled by stop() and mount_facet() separately, which are used + # inside fail() and fail_abort(). + # do_facet $facet $LCTL --device ${!svc} readonly do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}" $LCTL mark "local REPLAY BARRIER on ${!svc}" @@ -2182,14 +2352,32 @@ replay_barrier_nosync() { $LCTL mark "local REPLAY BARRIER on ${!svc}" } +# +# Get Lustre client uuid for a given Lustre mount point. +# +get_client_uuid() { + local mntpnt=${1:-$MOUNT} + + local name=$($LFS getname $mntpnt | cut -d' ' -f1) + local uuid=$($LCTL get_param -n llite.$name.uuid) + + echo -n $uuid +} + mds_evict_client() { - UUID=`lctl get_param -n mdc.${mds1_svc}-mdc-*.uuid` - do_facet mds1 "lctl set_param -n mdt.${mds1_svc}.evict_client $UUID" + local mntpnt=${1:-$MOUNT} + local uuid=$(get_client_uuid $mntpnt) + + do_facet $SINGLEMDS \ + "$LCTL set_param -n mdt.${mds1_svc}.evict_client $uuid" } ost_evict_client() { - UUID=`lctl get_param -n devices| grep ${ost1_svc}-osc- | egrep -v 'MDT' | awk '{print $5}'` - do_facet ost1 "lctl set_param -n obdfilter.${ost1_svc}.evict_client $UUID" + local mntpnt=${1:-$MOUNT} + local uuid=$(get_client_uuid $mntpnt) + + do_facet ost1 \ + "$LCTL set_param -n obdfilter.${ost1_svc}.evict_client $uuid" } fail() { @@ -2209,7 +2397,6 @@ fail_nodf() { fail_abort() { local facet=$1 stop $facet - refresh_disk ${facet} change_active $facet wait_for_facet $facet mount_facet $facet -o abort_recovery @@ -2235,42 +2422,43 @@ host_nids_address() { } h2name_or_ip() { - if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else - echo $1"@$2" - fi + if [ "$1" = "'*'" ]; then echo \'*\'; else + echo $1"@$2" + fi } h2ptl() { - if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else - ID=`xtprocadmin -n $1 2>/dev/null | egrep -v 'NID' | awk '{print $1}'` - if [ -z "$ID" ]; then - echo "Could not get a ptl id for $1..." - exit 1 - fi - echo $ID"@ptl" - fi + if [ "$1" = "'*'" ]; then echo \'*\'; else + ID=`xtprocadmin -n $1 2>/dev/null | egrep -v 'NID' | \ + awk '{print $1}'` + if [ -z "$ID" ]; then + echo "Could not get a ptl id for $1..." + exit 1 + fi + echo $ID"@ptl" + fi } declare -fx h2ptl h2tcp() { - h2name_or_ip "$1" "tcp" + h2name_or_ip "$1" "tcp" } declare -fx h2tcp h2elan() { - if [ "$1" = "client" -o "$1" = "'*'" ]; then echo \'*\'; else - if type __h2elan >/dev/null 2>&1; then - ID=$(__h2elan $1) - else - ID=`echo $1 | sed 's/[^0-9]*//g'` - fi - echo $ID"@elan" - fi + if [ "$1" = "'*'" ]; then echo \'*\'; else + if type __h2elan >/dev/null 2>&1; then + ID=$(__h2elan $1) + else + ID=`echo $1 | sed 's/[^0-9]*//g'` + fi + echo $ID"@elan" + fi } declare -fx h2elan h2o2ib() { - h2name_or_ip "$1" "o2ib" + h2name_or_ip "$1" "o2ib" } declare -fx h2o2ib @@ -2432,6 +2620,24 @@ facet_active_host() { fi } +# Get the passive failover partner host of facet. +facet_passive_host() { + local facet=$1 + [[ $facet = client ]] && return + + local host=${facet}_HOST + local failover_host=${facet}failover_HOST + local active_host=$(facet_active_host $facet) + + [[ -z ${!failover_host} || ${!failover_host} = ${!host} ]] && return + + if [[ $active_host = ${!host} ]]; then + echo -n ${!failover_host} + else + echo -n ${!host} + fi +} + change_active() { local facetlist=$1 local facet @@ -2617,13 +2823,26 @@ do_nodesv() { } add() { - local facet=$1 - shift - # make sure its not already running - stop ${facet} -f - rm -f $TMP/${facet}active - [[ $facet = mds1 ]] && combined_mgs_mds && rm -f $TMP/mgsactive - do_facet ${facet} $MKFS $* + local facet=$1 + shift + # make sure its not already running + stop ${facet} -f + rm -f $TMP/${facet}active + [[ $facet = mds1 ]] && combined_mgs_mds && rm -f $TMP/mgsactive + do_facet ${facet} $MKFS $* || return ${PIPESTATUS[0]} + + if [[ $(facet_fstype $facet) == zfs ]]; then + # + # After formatting a ZFS target, "cachefile=none" property will + # be set on the ZFS storage pool so that the pool is not + # automatically imported on system startup. And then the pool + # will be exported so as to leave the importing and exporting + # operations handled by mount_facet() and stop() separately. + # + refresh_partition_table $facet $(facet_vdevice $facet) + disable_zpool_cache $facet + export_zpool $facet + fi } ostdevname() { @@ -2883,6 +3102,9 @@ mkfs_opts() { if [ $fstype == ldiskfs ]; then fs_mkfs_opts+=${MDSJOURNALSIZE:+" -J size=$MDSJOURNALSIZE"} + if [ ! -z $EJOURNAL ]; then + fs_mkfs_opts+=${MDSJOURNALSIZE:+" device=$EJOURNAL"} + fi fs_mkfs_opts+=${MDSISIZE:+" -i $MDSISIZE"} fi fi @@ -3003,7 +3225,7 @@ writeconf_facet() { local dev=$2 stop ${facet} -f - rm -f ${facet}active + rm -f $TMP/${facet}active do_facet ${facet} "$TUNEFS --quiet --writeconf $dev" || return 1 return 0 } @@ -3246,26 +3468,20 @@ init_param_vars () { osc_ensure_active $SINGLEMDS $TIMEOUT osc_ensure_active client $TIMEOUT - local jobid_var - if [ -z "$(lctl get_param -n mdc.*.connect_flags | grep jobstats)" ]; then - jobid_var="none" - elif [ $JOBSTATS_AUTO -ne 0 ]; then - echo "enable jobstats, set job scheduler as $JOBID_VAR" - jobid_var=$JOBID_VAR - else - jobid_var=`$LCTL get_param -n jobid_var` - if [ $jobid_var != "disable" ]; then - echo "disable jobstats as required" - jobid_var="disable" - else - jobid_var="none" - fi - fi + if [ -n "$(lctl get_param -n mdc.*.connect_flags|grep jobstats)" ]; then + local current_jobid_var=$($LCTL get_param -n jobid_var) - if [ $jobid_var == $JOBID_VAR -o $jobid_var == "disable" ]; then - do_facet mgs $LCTL conf_param $FSNAME.sys.jobid_var=$jobid_var - wait_update $HOSTNAME "$LCTL get_param -n jobid_var" \ - $jobid_var || return 1 + if [ $JOBID_VAR = "existing" ]; then + echo "keeping jobstats as $current_jobid_var" + elif [ $current_jobid_var != $JOBID_VAR ]; then + echo "seting jobstats to $JOBID_VAR" + + set_conf_param_and_check client \ + "$LCTL get_param -n jobid_var" \ + "$FSNAME.sys.jobid_var" $JOBID_VAR + fi + else + echo "jobstats not supported by server" fi if [ $QUOTA_AUTO -ne 0 ]; then @@ -3375,10 +3591,17 @@ is_empty_dir() { # empty lustre filesystem may have empty directories lost+found and .lustre is_empty_fs() { + # exclude .lustre & lost+found [ $(find $1 -maxdepth 1 -name lost+found -o -name .lustre -prune -o \ -print | wc -l) = 1 ] || return 1 [ ! -d $1/lost+found ] || is_empty_dir $1/lost+found || return 1 - [ ! -d $1/.lustre ] || is_empty_dir $1/.lustre || return 1 + if [ $(lustre_version_code $SINGLEMDS) -gt $(version_code 2.4.0) ]; then + # exclude .lustre/fid (LU-2780) + [ $(find $1/.lustre -maxdepth 1 -name fid -prune -o \ + -print | wc -l) = 1 ] || return 1 + else + [ ! -d $1/.lustre ] || is_empty_dir $1/.lustre || return 1 + fi return 0 } @@ -3440,6 +3663,21 @@ check_and_setup_lustre() { set_default_debug_nodes $(comma_list $(nodes_list)) fi + if [ $(lower $OSD_TRACK_DECLARES_LBUG) == 'yes' ] ; then + local facets="" + [ "$(facet_fstype ost1)" = "ldiskfs" ] && + facets="$(get_facets OST)" + [ "$(facet_fstype mds1)" = "ldiskfs" ] && + facets="$facets,$(get_facets MDS)" + [ "$(facet_fstype mgs)" = "ldiskfs" ] && + facets="$facets,mgs" + local nodes="$(facets_hosts ${facets})" + if [ -n "$nodes" ] ; then + do_nodes $nodes "$LCTL set_param \ + osd-ldiskfs.track_declares_assert=1 || true" + fi + fi + init_gss if $GSS; then set_flavor_all $SEC @@ -3531,10 +3769,11 @@ run_e2fsck() { # verify a directory is shared among nodes. check_shared_dir() { local dir=$1 + local list=${2:-$(comma_list $(nodes_list))} [ -z "$dir" ] && return 1 - do_rpc_nodes "$(comma_list $(nodes_list))" check_logdir $dir - check_write_access $dir || return 1 + do_rpc_nodes "$list" check_logdir $dir + check_write_access $dir "$list" || return 1 return 0 } @@ -3769,7 +4008,7 @@ get_facets () { case $type in MGS ) list="$list $name";; - MDS|OST ) local count=${type}COUNT + MDS|OST|AGT ) local count=${type}COUNT for ((i=1; i<=${!count}; i++)) do list="$list ${name}$i" done;; @@ -4012,28 +4251,37 @@ log_trace_dump() { ################################## error_noexit() { - local TYPE=${TYPE:-"FAIL"} + local TYPE=${TYPE:-"FAIL"} - local dump=true - # do not dump logs if $1=false - if [ "x$1" = "xfalse" ]; then - shift - dump=false - fi + local dump=true + # do not dump logs if $1=false + if [ "x$1" = "xfalse" ]; then + shift + dump=false + fi - log " ${TESTSUITE} ${TESTNAME}: @@@@@@ ${TYPE}: $@ " - log_trace_dump - mkdir -p $LOGDIR - # We need to dump the logs on all nodes - if $dump; then - gather_logs $(comma_list $(nodes_list)) - fi + log " ${TESTSUITE} ${TESTNAME}: @@@@@@ ${TYPE}: $@ " + log_trace_dump + + mkdir -p $LOGDIR + # We need to dump the logs on all nodes + if $dump; then + gather_logs $(comma_list $(nodes_list)) + fi debugrestore [ "$TESTSUITELOG" ] && echo "$TESTSUITE: $TYPE: $TESTNAME $@" >> $TESTSUITELOG - echo "$@" > $LOGDIR/err + if [ -z "$*" ]; then + echo "error() without useful message, please fix" > $LOGDIR/err + else + if [[ `echo $TYPE | grep ^IGNORE` ]]; then + echo "$@" > $LOGDIR/ignore + else + echo "$@" > $LOGDIR/err + fi + fi } exit_status () { @@ -4045,21 +4293,22 @@ exit_status () { } error() { - error_noexit "$@" - exit 1 + error_noexit "$@" + exit 1 } error_exit() { - error "$@" + error "$@" } # use only if we are ignoring failures for this test, bugno required. # (like ALWAYS_EXCEPT, but run the test and ignore the results.) -# e.g. error_ignore 5494 "your message" +# e.g. error_ignore bz5494 "your message" or +# error_ignore LU-5494 "your message" error_ignore() { - local TYPE="IGNORE (bz$1)" - shift - error_noexit "$@" + local TYPE="IGNORE ($1)" + shift + error_noexit "$@" } error_and_remount() { @@ -4069,7 +4318,7 @@ error_and_remount() { } skip_env () { - $FAIL_ON_SKIP_ENV && error false $@ || skip $@ + $FAIL_ON_SKIP_ENV && error false $@ || skip $@ } skip() { @@ -4149,7 +4398,6 @@ run_test() { return $? fi LAST_SKIPPED="y" - echo -n "." return 0 fi @@ -4312,6 +4560,7 @@ run_one_logged() { local name=${TESTSUITE}.test_${1}.test_log.$(hostname -s).log local test_log=$LOGDIR/$name rm -rf $LOGDIR/err + rm -rf $LOGDIR/ignore rm -rf $LOGDIR/skip local SAVE_UMASK=`umask` umask 0022 @@ -4321,7 +4570,7 @@ run_one_logged() { (run_one $1 "$2") 2>&1 | tee -i $test_log local RC=${PIPESTATUS[0]} - [ $RC -ne 0 ] && [ ! -f $LOGDIR/err ] && \ + [ $RC -ne 0 ] && [ ! -f $LOGDIR/err ] && echo "test_$1 returned $RC" | tee $LOGDIR/err duration=$((`date +%s` - $BEFORE)) @@ -4329,6 +4578,8 @@ run_one_logged() { if [[ -f $LOGDIR/err ]]; then TEST_ERROR=$(cat $LOGDIR/err) + elif [[ -f $LOGDIR/ignore ]]; then + TEST_ERROR=$(cat $LOGDIR/ignore) elif [[ -f $LOGDIR/skip ]]; then TEST_ERROR=$(cat $LOGDIR/skip) fi @@ -4526,64 +4777,126 @@ local_mode () $(single_local_node $(comma_list $(nodes_list))) } -mdts_nodes () { - local MDSNODES - local NODES_sort - for num in `seq $MDSCOUNT`; do - MDSNODES="$MDSNODES $(facet_host mds$num)" - done - NODES_sort=$(for i in $MDSNODES; do echo $i; done | sort -u) - - echo $NODES_sort -} - remote_servers () { remote_ost && remote_mds } +# Get the active nodes for facets. facets_nodes () { - local facets=$1 - local nodes - local NODES_sort + local facets=$1 + local facet + local nodes + local nodes_sort + local i - for facet in ${facets//,/ }; do - if [ "$FAILURE_MODE" = HARD ]; then - nodes="$nodes $(facet_active_host $facet)" - else - nodes="$nodes $(facet_host $facet)" - fi - done - NODES_sort=$(for i in $nodes; do echo $i; done | sort -u) + for facet in ${facets//,/ }; do + nodes="$nodes $(facet_active_host $facet)" + done - echo $NODES_sort + nodes_sort=$(for i in $nodes; do echo $i; done | sort -u) + echo -n $nodes_sort } +# Get all of the active MDS nodes. +mdts_nodes () { + echo -n $(facets_nodes $(get_facets MDS)) +} + +# Get all of the active OSS nodes. osts_nodes () { - local facets=$(get_facets OST) - local nodes=$(facets_nodes $facets) + echo -n $(facets_nodes $(get_facets OST)) +} - echo $nodes +# Get all of the active AGT (HSM agent) nodes. +agts_nodes () { + echo -n $(facets_nodes $(get_facets AGT)) } +# Get all of the client nodes and active server nodes. nodes_list () { - # FIXME. We need a list of clients - local myNODES=$HOSTNAME - local myNODES_sort - - # CLIENTS (if specified) contains the local client - [ -n "$CLIENTS" ] && myNODES=${CLIENTS//,/ } + local nodes=$HOSTNAME + local nodes_sort + local i - if [ "$PDSH" -a "$PDSH" != "no_dsh" ]; then - myNODES="$myNODES $(facets_nodes $(get_facets))" - fi + # CLIENTS (if specified) contains the local client + [ -n "$CLIENTS" ] && nodes=${CLIENTS//,/ } - myNODES_sort=$(for i in $myNODES; do echo $i; done | sort -u) + if [ "$PDSH" -a "$PDSH" != "no_dsh" ]; then + nodes="$nodes $(facets_nodes $(get_facets))" + fi - echo $myNODES_sort + nodes_sort=$(for i in $nodes; do echo $i; done | sort -u) + echo -n $nodes_sort } +# Get all of the remote client nodes and remote active server nodes. remote_nodes_list () { - echo $(nodes_list) | sed -re "s/\<$HOSTNAME\>//g" + echo -n $(nodes_list) | sed -re "s/\<$HOSTNAME\>//g" +} + +# Get all of the MDS nodes, including active and passive nodes. +all_mdts_nodes () { + local host + local failover_host + local nodes + local nodes_sort + local i + + for i in $(seq $MDSCOUNT); do + host=mds${i}_HOST + failover_host=mds${i}failover_HOST + nodes="$nodes ${!host} ${!failover_host}" + done + + nodes_sort=$(for i in $nodes; do echo $i; done | sort -u) + echo -n $nodes_sort +} + +# Get all of the OSS nodes, including active and passive nodes. +all_osts_nodes () { + local host + local failover_host + local nodes + local nodes_sort + local i + + for i in $(seq $OSTCOUNT); do + host=ost${i}_HOST + failover_host=ost${i}failover_HOST + nodes="$nodes ${!host} ${!failover_host}" + done + + nodes_sort=$(for i in $nodes; do echo $i; done | sort -u) + echo -n $nodes_sort +} + +# Get all of the server nodes, including active and passive nodes. +all_server_nodes () { + local nodes + local nodes_sort + local i + + nodes="$mgs_HOST $mgsfailover_HOST $(all_mdts_nodes) $(all_osts_nodes)" + + nodes_sort=$(for i in $nodes; do echo $i; done | sort -u) + echo -n $nodes_sort +} + +# Get all of the client and server nodes, including active and passive nodes. +all_nodes () { + local nodes=$HOSTNAME + local nodes_sort + local i + + # CLIENTS (if specified) contains the local client + [ -n "$CLIENTS" ] && nodes=${CLIENTS//,/ } + + if [ "$PDSH" -a "$PDSH" != "no_dsh" ]; then + nodes="$nodes $(all_server_nodes)" + fi + + nodes_sort=$(for i in $nodes; do echo $i; done | sort -u) + echo -n $nodes_sort } init_clients_lists () { @@ -4664,10 +4977,11 @@ generate_machine_file() { } get_stripe () { - local file=$1/stripe - touch $file - $LFS getstripe -v $file || error - rm -f $file + local file=$1/stripe + + touch $file + $LFS getstripe -v $file || error "getstripe $file failed" + rm -f $file } setstripe_nfsserver () { @@ -4884,21 +5198,33 @@ calc_osc_kbytes () { $LCTL get_param -n osc.*[oO][sS][cC][-_][0-9a-f]*.$1 | calc_sum } -# save_lustre_params(node, parameter_mask) -# generate a stream of formatted strings ( =) +# save_lustre_params(comma separated facet list, parameter_mask) +# generate a stream of formatted strings ( =) save_lustre_params() { - local s - do_nodesv $1 "lctl get_param $2 | while read s; do echo \\\$s; done" + local facets=$1 + local facet + local nodes + local node + + for facet in ${facets//,/ }; do + node=$(facet_active_host $facet) + [[ *\ $node\ * = " $nodes " ]] && continue + nodes="$nodes $node" + + do_node $node "$LCTL get_param $2 | + while read s; do echo $facet \\\$s; done" + done } # restore lustre parameters from input stream, produces by save_lustre_params restore_lustre_params() { - local node - local name - local val - while IFS=" =" read node name val; do - do_node ${node//:/} "lctl set_param -n $name $val" - done + local facet + local name + local val + + while IFS=" =" read facet name val; do + do_facet $facet "$LCTL set_param -n $name $val" + done } check_catastrophe() { @@ -5125,7 +5451,7 @@ request_timeout () { echo $(( init_connect_timeout + at_min )) } -wait_osc_import_state() { +_wait_osc_import_state() { local facet=$1 local ost_facet=$2 local expected=$3 @@ -5142,7 +5468,7 @@ wait_osc_import_state() { #until list_param can return valid value. And also if there #are mulitple osc entries we should list all of them before #go to wait. - local params=$($LCTL list_param $param || true) + local params=$($LCTL list_param $param 2>/dev/null || true) while [ -z "$params" ]; do if [ $i -ge $maxtime ]; then echo "can't get $param by list_param in $maxtime secs" @@ -5156,10 +5482,10 @@ wait_osc_import_state() { fi sleep 1 i=$((i + 1)) - params=$($LCTL list_param $param || true) + params=$($LCTL list_param $param 2>/dev/null || true) done - if ! do_rpc_nodes "$(facet_host $facet)" \ + if ! do_rpc_nodes "$(facet_active_host $facet)" \ wait_import_state $expected "$params" $maxtime; then error "import is not in ${expected} state" return 1 @@ -5168,6 +5494,21 @@ wait_osc_import_state() { return 0 } +wait_osc_import_state() { + local facet=$1 + local ost_facet=$2 + local expected=$3 + local num + + if [[ $facet = mds ]]; then + for num in $(seq $MDSCOUNT); do + _wait_osc_import_state mds$num "$ost_facet" "$expected" + done + else + _wait_osc_import_state "$facet" "$ost_facet" "$expected" + fi +} + get_clientmdc_proc_path() { echo "${1}-mdc-*" } @@ -5179,7 +5520,9 @@ do_rpc_nodes () { [ -z "$list" ] && return 0 # Add paths to lustre tests for 32 and 64 bit systems. - local RPATH="PATH=$RLUSTRE/tests:/usr/lib/lustre/tests:/usr/lib64/lustre/tests:$PATH" + local LIBPATH="/usr/lib/lustre/tests:/usr/lib64/lustre/tests:" + local TESTPATH="$RLUSTRE/tests:" + local RPATH="PATH=${TESTPATH}${LIBPATH}${PATH}:/sbin:/bin:/usr/sbin:" do_nodesv $list "${RPATH} NAME=${NAME} sh rpc.sh $@ " } @@ -5717,19 +6060,20 @@ check_logdir() { } check_write_access() { - local dir=$1 - local node - local file + local dir=$1 + local list=${2:-$(comma_list $(nodes_list))} + local node + local file - for node in $(nodes_list); do - file=$dir/check_file.$(short_hostname $node) - if [[ ! -f "$file" ]]; then - # Logdir not accessible/writable from this node. - return 1 - fi - rm -f $file || return 1 - done - return 0 + for node in ${list//,/ }; do + file=$dir/check_file.$(short_hostname $node) + if [[ ! -f "$file" ]]; then + # Logdir not accessible/writable from this node. + return 1 + fi + rm -f $file || return 1 + done + return 0 } init_logging() { @@ -5808,6 +6152,33 @@ run_llverfs() llverfs $partial_arg $llverfs_opts $dir } +#Remove objects from OST +remove_ost_objects() { + local facet=$1 + local ostdev=$2 + local group=$3 + shift 3 + local objids="$@" + local mntpt=$(facet_mntpt $facet) + local opts=$OST_MOUNT_OPTS + local i + local rc + + echo "removing objects from $ostdev on $facet: $objids" + if ! test -b $ostdev; then + opts=$(csa_add "$opts" -o loop) + fi + mount -t $(facet_fstype $facet) $opts $ostdev $mntpt || + return $? + rc=0 + for i in $objids; do + rm $mntpt/O/$group/d$((i % 32))/$i || { rc=$?; break; } + done + umount -f $mntpt || return $? + return $rc +} + +#Remove files from MDT remove_mdt_files() { local facet=$1 local mdtdev=$2 @@ -5823,7 +6194,7 @@ remove_mdt_files() { fi mount -t $(facet_fstype $facet) $opts $mdtdev $mntpt || return $? - rc=0; + rc=0 for f in $files; do rm $mntpt/ROOT/$f || { rc=$?; break; } done @@ -5928,7 +6299,8 @@ large_xattr_enabled() { local mds_dev=$(mdsdevname ${SINGLEMDS//mds/}) - do_facet $SINGLEMDS "$DUMPE2FS -h $mds_dev 2>&1 | grep -q large_xattr" + do_facet $SINGLEMDS "$DUMPE2FS -h $mds_dev 2>&1 | + grep -E -q '(ea_inode|large_xattr)'" return ${PIPESTATUS[0]} } @@ -5967,30 +6339,33 @@ generate_string() { } reformat_external_journal() { + local facet=$1 + if [ ! -z ${EJOURNAL} ]; then - local rcmd="do_facet ${SINGLEMDS}" + local rcmd="do_facet $facet" - echo "reformat external journal on ${SINGLEMDS}:${EJOURNAL}" + echo "reformat external journal on $facet:${EJOURNAL}" ${rcmd} mke2fs -O journal_dev ${EJOURNAL} || return 1 fi } # MDT file-level backup/restore mds_backup_restore() { - local devname=$(mdsdevname ${SINGLEMDS//mds/}) + local facet=$1 + local igif=$2 + local devname=$(mdsdevname $(facet_number $facet)) local mntpt=$(facet_mntpt brpt) - local rcmd="do_facet ${SINGLEMDS}" + local rcmd="do_facet $facet" local metaea=${TMP}/backup_restore.ea local metadata=${TMP}/backup_restore.tgz local opts=${MDS_MOUNT_OPTS} - local svc=${SINGLEMDS}_svc - local igif=$1 + local svc=${facet}_svc if ! ${rcmd} test -b ${devname}; then opts=$(csa_add "$opts" -o loop) fi - echo "file-level backup/restore on ${SINGLEMDS}:${devname}" + echo "file-level backup/restore on $facet:${devname}" # step 1: build mount point ${rcmd} mkdir -p $mntpt @@ -6012,11 +6387,12 @@ mds_backup_restore() { # step 6: umount ${rcmd} umount -d $mntpt || return 4 # step 7: reformat external journal if needed - reformat_external_journal || return 5 + reformat_external_journal $facet || return 5 # step 8: reformat dev echo "reformat new device" - add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS}) --backfstype ldiskfs \ - --reformat $devname > /dev/null || return 6 + add $facet $(mkfs_opts $facet ${devname}) --backfstype ldiskfs \ + --reformat ${devname} $(mdsvdevname $(facet_number $facet)) \ + > /dev/null || exit 6 # step 9: mount dev ${rcmd} mount -t ldiskfs $opts $devname $mntpt || return 7 # step 10: restore metadata @@ -6038,17 +6414,18 @@ mds_backup_restore() { # remove OI files mds_remove_ois() { - local devname=$(mdsdevname ${SINGLEMDS//mds/}) + local facet=$1 + local idx=$2 + local devname=$(mdsdevname $(facet_number $facet)) local mntpt=$(facet_mntpt brpt) - local rcmd="do_facet ${SINGLEMDS}" - local idx=$1 + local rcmd="do_facet $facet" local opts=${MDS_MOUNT_OPTS} if ! ${rcmd} test -b ${devname}; then opts=$(csa_add "$opts" -o loop) fi - echo "remove OI files: idx=${idx}" + echo "removing OI files on $facet: idx=${idx}" # step 1: build mount point ${rcmd} mkdir -p $mntpt @@ -6080,41 +6457,28 @@ generate_logname() { echo "$TESTLOG_PREFIX.$TESTNAME.$logname.$(hostname -s).log" } -# mkdir directory on different MDTs +# make directory on different MDTs test_mkdir() { local option local parent local child local path - local dir local rc=0 - if [ $# -eq 2 ]; then - option=$1 - path=$2 - else - path=$1 - fi - - child=${path##*/} - parent=${path%/*} + case $# in + 1) path=$1;; + 2) option=$1 + path=$2;; + *) error "Only creating single directory is supported";; + esac - if [ "$parent" == "$child" ]; then - parent=$(pwd) - fi + child=$(basename $path) + parent=$(dirname $path) - if [ "$option" == "-p" -a -d ${parent}/${child} ]; then + if [ "$option" == "-p" -a -d $parent/$child ]; then return $rc fi - # it needs to check whether there is further / in child - dir=$(echo $child | awk -F '/' '{print $2}') - if [ ! -z "$dir" ]; then - local subparent=$(echo $child | awk -F '/' '{ print $1 }') - parent=${parent}"/"${subparent} - child=$dir - fi - if [ ! -d ${parent} ]; then if [ "$option" == "-p" ]; then mkdir -p ${parent} @@ -6124,19 +6488,18 @@ test_mkdir() { fi if [ $MDSCOUNT -le 1 ]; then - mkdir $option ${parent}/${child} || rc=$? + mkdir $option $parent/$child || rc=$? else local mdt_idx=$($LFS getstripe -M $parent) + local test_num=$(echo $testnum | sed -e 's/[^0-9]*//g') if [ "$mdt_idx" -ne 0 ]; then - mkdir $option ${parent}/${child} || rc=$? - return $rc + mkdir $option $parent/$child || rc=$? + else + mdt_idx=$((test_num % MDSCOUNT)) + echo "mkdir $mdt_idx for $parent/$child" + $LFS setdirstripe -i $mdt_idx $parent/$child || rc=$? fi - - local test_num=$(echo $testnum | sed -e 's/[^0-9]*//g') - local mdt_idx=$((test_num % MDSCOUNT)) - echo "mkdir $mdt_idx for ${parent}/${child}" - $LFS setdirstripe -i $mdt_idx ${parent}/${child} || rc=$? fi return $rc }