X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=9caa3f7e0d84f7ded04c30d8b8ec5e814059388a;hp=0a786497e4bb04f7f1222521aac4634cb9fa53ee;hb=73ecd83a24e220b8097facf5bf3f5d93f523702c;hpb=5446b9b6246c9721394ab3aefb94779c899927de diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 0a78649..9caa3f7 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -22,8 +22,10 @@ export SK_UNIQUE_NM=${SK_UNIQUE_NM:-false} export SK_S2S=${SK_S2S:-false} export SK_S2SNM=${SK_S2SNM:-TestFrameNM} export SK_S2SNMCLI=${SK_S2SNMCLI:-TestFrameNMCli} +export SK_SKIPFIRST=${SK_SKIPFIRST:-true} export IDENTITY_UPCALL=default export QUOTA_AUTO=1 +export FLAKEY=${FLAKEY:-true} # specify environment variable containing batch job name for server statistics export JOBID_VAR=${JOBID_VAR:-"procname_uid"} # or "existing" or "disable" @@ -44,7 +46,7 @@ LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} . $LUSTRE/tests/functions.sh . $LUSTRE/tests/yaml.sh -export LD_LIBRARY_PATH=${LUSTRE}/utils:${LD_LIBRARY_PATH} +export LD_LIBRARY_PATH=${LUSTRE}/utils/.libs:${LUSTRE}/utils:${LD_LIBRARY_PATH} LUSTRE_TESTS_CFG_DIR=${LUSTRE_TESTS_CFG_DIR:-${LUSTRE}/tests/cfg} @@ -154,6 +156,8 @@ init_test_env() { export RPC_MODE=${RPC_MODE:-false} export DO_CLEANUP=${DO_CLEANUP:-true} export KEEP_ZPOOL=${KEEP_ZPOOL:-false} + export CLEANUP_DM_DEV=false + export PAGE_SIZE=$(get_page_size client) export MKE2FS=$MKE2FS if [ -z "$MKE2FS" ]; then @@ -272,6 +276,8 @@ init_test_env() { SETSTRIPE=${SETSTRIPE:-"$LFS setstripe"} GETSTRIPE=${GETSTRIPE:-"$LFS getstripe"} + export PERM_CMD=${PERM_CMD:-"$LCTL conf_param"} + export L_GETIDENTITY=${L_GETIDENTITY:-"$LUSTRE/utils/l_getidentity"} if [ ! -f "$L_GETIDENTITY" ]; then if `which l_getidentity > /dev/null 2>&1`; then @@ -292,7 +298,7 @@ init_test_env() { export LUSTRE_RMMOD=${LUSTRE_RMMOD:-$LUSTRE/scripts/lustre_rmmod} [ ! -f "$LUSTRE_RMMOD" ] && export LUSTRE_RMMOD=$(which lustre_rmmod 2> /dev/null) - export LUSTRE_ROUTES_COVERSION=${LUSTRE_ROUTES_CONVERSION:-$LUSTRE/scripts/lustre_routes_conversion} + export LUSTRE_ROUTES_CONVERSION=${LUSTRE_ROUTES_CONVERSION:-$LUSTRE/scripts/lustre_routes_conversion} [ ! -f "$LUSTRE_ROUTES_CONVERSION" ] && export LUSTRE_ROUTES_CONVERSION=$(which lustre_routes_conversion 2> /dev/null) export LFS_MIGRATE=${LFS_MIGRATE:-$LUSTRE/scripts/lfs_migrate} @@ -302,6 +308,10 @@ init_test_env() { [ ! -f "$LR_READER" ] && export LR_READER=$(which lr_reader 2> /dev/null) [ -z "$LR_READER" ] && export LR_READER="/usr/sbin/lr_reader" + export LSOM_SYNC=${LSOM_SYNC:-"$LUSTRE/utils/llsom_sync"} + [ ! -f "$LSOM_SYNC" ] && + export LSOM_SYNC=$(which llsom_sync 2> /dev/null) + [ -z "$LSOM_SYNC" ] && export LSOM_SYNC="/usr/sbin/llsom_sync" export NAME=${NAME:-local} export LGSSD=${LGSSD:-"$LUSTRE/utils/gss/lgssd"} [ "$GSS_PIPEFS" = "true" ] && [ ! -f "$LGSSD" ] && @@ -315,6 +325,10 @@ init_test_env() { export LDEV=${LDEV:-"$LUSTRE/scripts/ldev"} [ ! -f "$LDEV" ] && export LDEV=$(which ldev 2> /dev/null) + export DMSETUP=${DMSETUP:-dmsetup} + export DM_DEV_PATH=${DM_DEV_PATH:-/dev/mapper} + export LOSETUP=${LOSETUP:-losetup} + if [ "$ACCEPTOR_PORT" ]; then export PORT_OPT="--port $ACCEPTOR_PORT" fi @@ -353,6 +367,7 @@ init_test_env() { export RLUSTRE=${RLUSTRE:-$LUSTRE} export RPWD=${RPWD:-$PWD} export I_MOUNTED=${I_MOUNTED:-"no"} + export AUSTER_CLEANUP=${AUSTER_CLEANUP:-false} if [ ! -f /lib/modules/$(uname -r)/kernel/fs/lustre/mdt.ko -a \ ! -f /lib/modules/$(uname -r)/updates/kernel/fs/lustre/mdt.ko -a \ ! -f /lib/modules/$(uname -r)/extra/kernel/fs/lustre/mdt.ko -a \ @@ -500,62 +515,91 @@ lustre_insmod() { # /etc/modprobe.conf, from /etc/modprobe.d/Lustre, or else none will be used. # load_module() { - local optvar - EXT=".ko" - module=$1 - shift - BASE=$(basename $module $EXT) - - module_loaded ${BASE} && return - - # If no module arguments were passed, get them from $MODOPTS_, - # else from modprobe.conf - if [ $# -eq 0 ]; then - # $MODOPTS_; we could use associative arrays, but that's not in - # Bash until 4.x, so we resort to eval. - optvar="MODOPTS_$(basename $module | tr a-z A-Z)" - eval set -- \$$optvar - if [ $# -eq 0 -a -n "$MODPROBECONF" ]; then - # Nothing in $MODOPTS_; try modprobe.conf - local opt - opt=$(awk -v var="^options $BASE" '$0 ~ var \ - {gsub("'"options $BASE"'",""); print}' $MODPROBECONF) - set -- $(echo -n $opt) - - # Ensure we have accept=all for lnet - if [ $(basename $module) = lnet ]; then - # OK, this is a bit wordy... - local arg accept_all_present=false - - for arg in "$@"; do - [ "$arg" = accept=all ] && \ - accept_all_present=true - done - $accept_all_present || set -- "$@" accept=all + local module=$1 # '../libcfs/libcfs/libcfs', 'obdclass/obdclass', ... + shift + local ext=".ko" + local base=$(basename $module $ext) + local path + local -A module_is_loaded_aa + local optvar + local mod + + for mod in $(lsmod | awk '{ print $1; }'); do + module_is_loaded_aa[${mod//-/_}]=true + done + + module_is_loaded() { + ${module_is_loaded_aa[${1//-/_}]:-false} + } + + if module_is_loaded $base; then + return + fi + + if [[ -f $LUSTRE/$module$ext ]]; then + path=$LUSTRE/$module$ext + elif [[ "$base" == lnet_selftest ]] && + [[ -f $LUSTRE/../lnet/selftest/$base$ext ]]; then + path=$LUSTRE/../lnet/selftest/$base$ext + else + path='' + fi + + if [[ -n "$path" ]]; then + # Try to load any non-Lustre modules that $module depends on. + for mod in $(modinfo --field=depends $path | tr ',' ' '); do + if ! module_is_loaded $mod; then + modprobe $mod + fi + done + fi + + # If no module arguments were passed then get them from + # $MODOPTS_, otherwise from modprobe.conf. + if [ $# -eq 0 ]; then + # $MODOPTS_; we could use associative arrays, but that's + # not in Bash until 4.x, so we resort to eval. + optvar="MODOPTS_$(basename $module | tr a-z A-Z)" + eval set -- \$$optvar + if [ $# -eq 0 -a -n "$MODPROBECONF" ]; then + # Nothing in $MODOPTS_; try modprobe.conf + local opt + opt=$(awk -v var="^options $base" '$0 ~ var \ + {gsub("'"options $base"'",""); print}' \ + $MODPROBECONF) + set -- $(echo -n $opt) + + # Ensure we have accept=all for lnet + if [[ "$base" == lnet ]]; then + # OK, this is a bit wordy... + local arg accept_all_present=false + + for arg in "$@"; do + [[ "$arg" == accept=all ]] && + accept_all_present=true + done + + $accept_all_present || set -- "$@" accept=all + fi + + export $optvar="$*" fi - export $optvar="$*" - fi - fi + fi - [ $# -gt 0 ] && echo "${module} options: '$*'" + [ $# -gt 0 ] && echo "${module} options: '$*'" # Note that insmod will ignore anything in modprobe.conf, which is why - # we're passing options on the command-line. - if [[ "$BASE" == "lnet_selftest" ]] && - [[ -f ${LUSTRE}/../lnet/selftest/${module}${EXT} ]]; then - lustre_insmod ${LUSTRE}/../lnet/selftest/${module}${EXT} - elif [[ -f ${LUSTRE}/${module}${EXT} ]]; then - [[ "$BASE" != "ptlrpc_gss" ]] || modprobe sunrpc - lustre_insmod ${LUSTRE}/${module}${EXT} "$@" - else - # must be testing a "make install" or "rpm" installation - # note failed to load ptlrpc_gss is considered not fatal - if [[ "$BASE" == "ptlrpc_gss" ]]; then - modprobe $BASE "$@" 2>/dev/null || - echo "gss/krb5 is not supported" - else - modprobe $BASE "$@" + # we're passing options on the command-line. If $path does not exist + # then we must be testing a "make install" or"rpm" installation. Also + # note that failing to load ptlrpc_gss is not considered fatal. + if [[ -n "$path" ]]; then + lustre_insmod $path "$@" + elif [[ "$base" == ptlrpc_gss ]]; then + if ! modprobe $base "$@" 2>/dev/null; then + echo "gss/krb5 is not supported" fi + else + modprobe $base "$@" fi } @@ -566,6 +610,15 @@ load_modules_local() { return 0 fi + # Create special udev test rules on every node + if [ -f $LUSTRE/lustre/conf/99-lustre.rules ]; then { + sed -e 's|/usr/sbin/lctl|$LCTL|g' $LUSTRE/lustre/conf/99-lustre.rules > /etc/udev/rules.d/99-lustre-test.rules + } else { + echo "SUBSYSTEM==\"lustre\", ACTION==\"change\", ENV{PARAM}==\"?*\", RUN+=\"$LCTL set_param '\$env{PARAM}=\$env{SETTING}'\"" > /etc/udev/rules.d/99-lustre-test.rules + } fi + udevadm control --reload-rules + udevadm trigger + echo Loading modules from $LUSTRE local ncpus @@ -628,23 +681,12 @@ load_modules_local() { load_module mgc/mgc load_module obdecho/obdecho if ! client_only; then - SYMLIST=/proc/kallsyms - grep -q crc16 $SYMLIST || - { modprobe crc16 2>/dev/null || true; } - grep -q -w jbd2 $SYMLIST || - { modprobe jbd2 2>/dev/null || true; } load_module lfsck/lfsck [ "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS if [[ $(node_fstypes $HOSTNAME) == *zfs* ]]; then - lsmod | grep zfs >&/dev/null || modprobe zfs load_module osd-zfs/osd_zfs - fi - if [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then - grep -q exportfs_decode_fh $SYMLIST || - { modprobe exportfs 2> /dev/null || true; } - grep -q -w mbcache $SYMLIST || - { modprobe mbcache 2>/dev/null || true; } + elif [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then load_module ../ldiskfs/ldiskfs load_module osd-ldiskfs/osd_ldiskfs fi @@ -722,12 +764,20 @@ unload_modules() { $LUSTRE_RMMOD ldiskfs || return 2 + [ -f /etc/udev/rules.d/99-lustre-test.rules ] && + rm /etc/udev/rules.d/99-lustre-test.rules + udevadm control --reload-rules + udevadm trigger + if $LOAD_MODULES_REMOTE; then local list=$(comma_list $(remote_nodes_list)) if [ -n "$list" ]; then echo "unloading modules on: '$list'" do_rpc_nodes "$list" $LUSTRE_RMMOD ldiskfs do_rpc_nodes "$list" check_mem_leak + do_rpc_nodes "$list" "rm /etc/udev/rules.d/99-lustre-test.rules" + do_rpc_nodes "$list" "udevadm control --reload-rules" + do_rpc_nodes "$list" "udevadm trigger" fi fi @@ -750,11 +800,13 @@ fs_log_size() { case $(facet_fstype $facet) in ldiskfs) size=50;; # largest seen is 44, leave some headroom - zfs) size=$(lctl get_param osc.$FSNAME-OST*.import | - awk '/grant_block_size:/ { print $2 * 2; exit; }');; + # grant_block_size is in bytes, allow at least 2x max blocksize + zfs) size=$(lctl get_param osc.$FSNAME*.import | + awk '/grant_block_size:/ {print $2/512; exit;}') + ;; esac - echo -n $size + echo -n $((size * MDSCOUNT)) } fs_inode_ksize() { @@ -910,7 +962,11 @@ init_gss() { # security ctx config for keyring SK_NO_KEY=false mkdir -p $SK_OM_PATH - mount -o bind $SK_OM_PATH /etc/request-key.d/ + if grep -q request-key /proc/mounts > /dev/null; then + echo "SSK: Request key already mounted." + else + mount -o bind $SK_OM_PATH /etc/request-key.d/ + fi local lgssc_conf_line='create lgssc * * ' lgssc_conf_line+=$(which lgss_keyring) lgssc_conf_line+=' %o %k %t %d %c %u %g %T %P %S' @@ -986,6 +1042,8 @@ init_gss() { -m $SK_PATH/$FSNAME-nmclient.key \ >/dev/null 2>&1" fi + fi + if $GSS_SK; then # mount options for servers and clients MGS_MOUNT_OPTS=$(add_sk_mntflag $MGS_MOUNT_OPTS) MDS_MOUNT_OPTS=$(add_sk_mntflag $MDS_MOUNT_OPTS) @@ -1019,9 +1077,12 @@ cleanup_sk() { $RPC_MODE || echo "Cleaning up Shared Key.." do_nodes $(comma_list $(all_nodes)) "rm -f \ $SK_PATH/$FSNAME*.key $SK_PATH/nodemap/$FSNAME*.key" + do_nodes $(comma_list $(all_nodes)) "keyctl show | \ + awk '/lustre/ { print \\\$1 }' | xargs -IX keyctl unlink X" # Remove the mount and clean up the files we added to SK_PATH - do_nodes $(comma_list $(all_nodes)) "umount \ - /etc/request-key.d/" + do_nodes $(comma_list $(all_nodes)) "while grep -q \ + request-key.d /proc/mounts; do umount \ + /etc/request-key.d/; done" do_nodes $(comma_list $(all_nodes)) "rm -f \ $SK_OM_PATH/lgssc.conf" do_nodes $(comma_list $(all_nodes)) "rmdir $SK_OM_PATH" @@ -1467,7 +1528,8 @@ mount_facets () { [ $RC -eq 0 ] && continue if [ "$TESTSUITE.$TESTNAME" = "replay-dual.test_0a" ]; then - skip "Restart of $facet failed!." && touch $LU482_FAILED + skip_noexit "Restart of $facet failed!." && + touch $LU482_FAILED else error "Restart of $facet failed!" fi @@ -1495,50 +1557,385 @@ csa_add() { echo -n "$opts" } +# +# Associate loop device with a given regular file. +# Return the loop device. +# +setup_loop_device() { + local facet=$1 + local file=$2 + + do_facet $facet "loop_dev=\\\$($LOSETUP -j $file | cut -d : -f 1); + if [[ -z \\\$loop_dev ]]; then + loop_dev=\\\$($LOSETUP -f); + $LOSETUP \\\$loop_dev $file || loop_dev=; + fi; + echo -n \\\$loop_dev" +} + +# +# Detach a loop device. +# +cleanup_loop_device() { + local facet=$1 + local loop_dev=$2 + + do_facet $facet "! $LOSETUP $loop_dev >/dev/null 2>&1 || + $LOSETUP -d $loop_dev" +} + +# +# Check if a given device is a block device. +# +is_blkdev() { + local facet=$1 + local dev=$2 + local size=${3:-""} + + [[ -n "$dev" ]] || return 1 + do_facet $facet "test -b $dev" || return 1 + if [[ -n "$size" ]]; then + local in=$(do_facet $facet "dd if=$dev of=/dev/null bs=1k \ + count=1 skip=$size 2>&1" | + awk '($3 == "in") { print $1 }') + [[ "$in" = "1+0" ]] || return 1 + fi +} + +# +# Check if a given device is a device-mapper device. +# +is_dm_dev() { + local facet=$1 + local dev=$2 + + [[ -n "$dev" ]] || return 1 + do_facet $facet "$DMSETUP status $dev >/dev/null 2>&1" +} + +# +# Check if a given device is a device-mapper flakey device. +# +is_dm_flakey_dev() { + local facet=$1 + local dev=$2 + local type + + [[ -n "$dev" ]] || return 1 + + type=$(do_facet $facet "$DMSETUP status $dev 2>&1" | + awk '{print $3}') + [[ $type = flakey ]] && return 0 || return 1 +} + +# +# Check if device-mapper flakey device is supported by the kernel +# of $facet node or not. +# +dm_flakey_supported() { + local facet=$1 + + $FLAKEY || return 1 + do_facet $facet "modprobe dm-flakey; + $DMSETUP targets | grep -q flakey" &> /dev/null +} + +# +# Get the device-mapper flakey device name of a given facet. +# +dm_facet_devname() { + local facet=$1 + [[ $facet = mgs ]] && combined_mgs_mds && facet=mds1 + + echo -n ${facet}_flakey +} + +# +# Get the device-mapper flakey device of a given facet. +# A device created by dmsetup will appear as /dev/mapper/. +# +dm_facet_devpath() { + local facet=$1 + + echo -n $DM_DEV_PATH/$(dm_facet_devname $facet) +} + +# +# Set a device-mapper device with a new table. +# +# The table has the following format: +# +# +# flakey includes: +# \ +# [ []] +# +# linear includes: +# +# +dm_set_dev_table() { + local facet=$1 + local dm_dev=$2 + local target_type=$3 + local num_sectors + local real_dev + local tmp + local table + + read tmp num_sectors tmp real_dev tmp \ + <<< $(do_facet $facet "$DMSETUP table $dm_dev") + + case $target_type in + flakey) + table="0 $num_sectors flakey $real_dev 0 0 1800 1 drop_writes" + ;; + linear) + table="0 $num_sectors linear $real_dev 0" + ;; + *) error "invalid target type $target_type" ;; + esac + + do_facet $facet "$DMSETUP suspend --nolockfs --noflush $dm_dev" || + error "failed to suspend $dm_dev" + do_facet $facet "$DMSETUP load $dm_dev --table \\\"$table\\\"" || + error "failed to load $target_type table into $dm_dev" + do_facet $facet "$DMSETUP resume $dm_dev" || + error "failed to resume $dm_dev" +} + +# +# Set a device-mapper flakey device as "read-only" by using the "drop_writes" +# feature parameter. +# +# drop_writes: +# All write I/O is silently ignored. +# Read I/O is handled correctly. +# +dm_set_dev_readonly() { + local facet=$1 + local dm_dev=${2:-$(dm_facet_devpath $facet)} + + dm_set_dev_table $facet $dm_dev flakey +} + +# +# Set a device-mapper device to traditional linear mapping mode. +# +dm_clear_dev_readonly() { + local facet=$1 + local dm_dev=${2:-$(dm_facet_devpath $facet)} + + dm_set_dev_table $facet $dm_dev linear +} + +# +# Set the device of a given facet as "read-only". +# +set_dev_readonly() { + local facet=$1 + local svc=${facet}_svc + + if [[ $(facet_fstype $facet) = zfs ]] || + ! dm_flakey_supported $facet; then + do_facet $facet $LCTL --device ${!svc} readonly + else + dm_set_dev_readonly $facet + fi +} + +# +# Get size in 512-byte sectors (BLKGETSIZE64 / 512) of a given device. +# +get_num_sectors() { + local facet=$1 + local dev=$2 + local num_sectors + + num_sectors=$(do_facet $facet "blockdev --getsz $dev 2>/dev/null") + [[ ${PIPESTATUS[0]} = 0 && -n "$num_sectors" ]] || num_sectors=0 + echo -n $num_sectors +} + +# +# Create a device-mapper device with a given block device or regular file (will +# be associated with loop device). +# Return the full path of the device-mapper device. +# +dm_create_dev() { + local facet=$1 + local real_dev=$2 # destination device + local dm_dev_name=${3:-$(dm_facet_devname $facet)} # device name + local dm_dev=$DM_DEV_PATH/$dm_dev_name # device-mapper device + + # check if the device-mapper device to be created already exists + if is_dm_dev $facet $dm_dev; then + # if the existing device was set to "read-only", then clear it + ! is_dm_flakey_dev $facet $dm_dev || + dm_clear_dev_readonly $facet $dm_dev + + echo -n $dm_dev + return 0 + fi + + # check if the destination device is a block device, and if not, + # associate it with a loop device + is_blkdev $facet $real_dev || + real_dev=$(setup_loop_device $facet $real_dev) + [[ -n "$real_dev" ]] || { echo -n $real_dev; return 2; } + + # now create the device-mapper device + local num_sectors=$(get_num_sectors $facet $real_dev) + local table="0 $num_sectors linear $real_dev 0" + local rc=0 + + do_facet $facet "$DMSETUP create $dm_dev_name --table \\\"$table\\\"" || + { rc=${PIPESTATUS[0]}; dm_dev=; } + do_facet $facet "$DMSETUP mknodes >/dev/null 2>&1" + + echo -n $dm_dev + return $rc +} + +# +# Map the facet name to its device variable name. +# +facet_device_alias() { + local facet=$1 + local dev_alias=$facet + + case $facet in + fs2mds) dev_alias=mds1_2 ;; + fs2ost) dev_alias=ost1_2 ;; + fs3ost) dev_alias=ost2_2 ;; + *) ;; + esac + + echo -n $dev_alias +} + +# +# Save the original value of the facet device and export the new value. +# +export_dm_dev() { + local facet=$1 + local dm_dev=$2 + + local active_facet=$(facet_active $facet) + local dev_alias=$(facet_device_alias $active_facet) + local dev_name=${dev_alias}_dev + local dev=${!dev_name} + + if [[ $active_facet = $facet ]]; then + local failover_dev=${dev_alias}failover_dev + if [[ ${!failover_dev} = $dev ]]; then + eval export ${failover_dev}_saved=$dev + eval export ${failover_dev}=$dm_dev + fi + else + dev_alias=$(facet_device_alias $facet) + local facet_dev=${dev_alias}_dev + if [[ ${!facet_dev} = $dev ]]; then + eval export ${facet_dev}_saved=$dev + eval export ${facet_dev}=$dm_dev + fi + fi + + eval export ${dev_name}_saved=$dev + eval export ${dev_name}=$dm_dev +} + +# +# Restore the saved value of the facet device. +# +unexport_dm_dev() { + local facet=$1 + + [[ $facet = mgs ]] && combined_mgs_mds && facet=mds1 + local dev_alias=$(facet_device_alias $facet) + + local saved_dev=${dev_alias}_dev_saved + [[ -z ${!saved_dev} ]] || + eval export ${dev_alias}_dev=${!saved_dev} + + saved_dev=${dev_alias}failover_dev_saved + [[ -z ${!saved_dev} ]] || + eval export ${dev_alias}failover_dev=${!saved_dev} +} + +# +# Remove a device-mapper device. +# If the destination device is a loop device, then also detach it. +# +dm_cleanup_dev() { + local facet=$1 + local dm_dev=${2:-$(dm_facet_devpath $facet)} + local major + local minor + + is_dm_dev $facet $dm_dev || return 0 + + read major minor <<< $(do_facet $facet "$DMSETUP table $dm_dev" | + awk '{ print $4 }' | awk -F: '{ print $1" "$2 }') + + do_facet $facet "$DMSETUP remove $dm_dev" + do_facet $facet "$DMSETUP mknodes >/dev/null 2>&1" + + unexport_dm_dev $facet + + # detach a loop device + [[ $major -ne 7 ]] || cleanup_loop_device $facet /dev/loop$minor + + # unload dm-flakey module + do_facet $facet "modprobe -r dm-flakey" || true +} + mount_facet() { local facet=$1 shift - local dev=$(facet_active $facet)_dev + local active_facet=$(facet_active $facet) + local dev_alias=$(facet_device_alias $active_facet) + local dev=${dev_alias}_dev local opt=${facet}_opt local mntpt=$(facet_mntpt $facet) local opts="${!opt} $@" local fstype=$(facet_fstype $facet) local devicelabel + local dm_dev=${!dev} module_loaded lustre || load_modules - if [ $(facet_fstype $facet) == ldiskfs ] && - ! do_facet $facet test -b ${!dev}; then - opts=$(csa_add "$opts" -o loop) - fi + case $fstype in + ldiskfs) + if dm_flakey_supported $facet; then + dm_dev=$(dm_create_dev $facet ${!dev}) + [[ -n "$dm_dev" ]] || dm_dev=${!dev} + fi - if [[ $(facet_fstype $facet) == zfs ]]; then + is_blkdev $facet $dm_dev || opts=$(csa_add "$opts" -o loop) + + devicelabel=$(do_facet ${facet} "$E2LABEL $dm_dev");; + zfs) # import ZFS storage pool import_zpool $facet || return ${PIPESTATUS[0]} - fi - case $fstype in - ldiskfs) - devicelabel=$(do_facet ${facet} "$E2LABEL ${!dev}");; - zfs) devicelabel=$(do_facet ${facet} "$ZFS get -H -o value \ - lustre:svname ${!dev}");; + lustre:svname $dm_dev");; *) error "unknown fstype!";; esac - echo "Starting ${facet}: $opts ${!dev} $mntpt" + echo "Starting ${facet}: $opts $dm_dev $mntpt" # for testing LU-482 error handling in mount_facets() and test_0a() if [ -f $TMP/test-lu482-trigger ]; then RC=2 else - do_facet ${facet} "mkdir -p $mntpt; $MOUNT_CMD $opts \ - ${!dev} $mntpt" + do_facet ${facet} \ + "mkdir -p $mntpt; $MOUNT_CMD $opts $dm_dev $mntpt" RC=${PIPESTATUS[0]} fi if [ $RC -ne 0 ]; then - echo "Start of ${!dev} on ${facet} failed ${RC}" + echo "Start of $dm_dev on ${facet} failed ${RC}" return $RC fi @@ -1555,19 +1952,19 @@ mount_facet() { fi if [[ $opts =~ .*nosvc.* ]]; then - echo "Start ${!dev} without service" + echo "Start $dm_dev without service" else case $fstype in ldiskfs) - wait_update_facet ${facet} "$E2LABEL ${!dev} \ + wait_update_facet ${facet} "$E2LABEL $dm_dev \ 2>/dev/null | grep -E ':[a-zA-Z]{3}[0-9]{4}'" \ - "" || error "${!dev} failed to initialize!";; + "" || error "$dm_dev failed to initialize!";; zfs) wait_update_facet ${facet} "$ZFS get -H -o value \ - lustre:svname ${!dev} 2>/dev/null | \ + lustre:svname $dm_dev 2>/dev/null | \ grep -E ':[a-zA-Z]{3}[0-9]{4}'" "" || - error "${!dev} failed to initialize!";; + error "$dm_dev failed to initialize!";; *) error "unknown fstype!";; @@ -1581,11 +1978,13 @@ mount_facet() { fi - label=$(devicelabel ${facet} ${!dev}) - [ -z "$label" ] && echo no label for ${!dev} && exit 1 + label=$(devicelabel ${facet} $dm_dev) + [ -z "$label" ] && echo no label for $dm_dev && exit 1 eval export ${facet}_svc=${label} echo Started ${label} + export_dm_dev $facet $dm_dev + return $RC } @@ -1595,14 +1994,16 @@ start() { shift local device=$1 shift - eval export ${facet}_dev=${device} + local dev_alias=$(facet_device_alias $facet) + + eval export ${dev_alias}_dev=${device} eval export ${facet}_opt=\"$@\" - local varname=${facet}failover_dev + local varname=${dev_alias}failover_dev if [ -n "${!varname}" ] ; then - eval export ${facet}failover_dev=${!varname} + eval export ${dev_alias}failover_dev=${!varname} else - eval export ${facet}failover_dev=$device + eval export ${dev_alias}failover_dev=$device fi local mntpt=$(facet_mntpt $facet) @@ -1621,18 +2022,18 @@ start() { } stop() { - local running - local facet=$1 - shift - local HOST=`facet_active_host $facet` - [ -z $HOST ] && echo stop: no host for $facet && return 0 - - local mntpt=$(facet_mntpt $facet) - running=$(do_facet ${facet} "grep -c $mntpt' ' /proc/mounts") || true - if [ ${running} -ne 0 ]; then - echo "Stopping $mntpt (opts:$@) on $HOST" - do_facet ${facet} $UMOUNT $@ $mntpt - fi + local running + local facet=$1 + shift + local HOST=$(facet_active_host $facet) + [[ -z $HOST ]] && echo stop: no host for $facet && return 0 + + local mntpt=$(facet_mntpt $facet) + running=$(do_facet ${facet} "grep -c $mntpt' ' /proc/mounts || true") + if [ ${running} -ne 0 ]; then + echo "Stopping $mntpt (opts:$@) on $HOST" + do_facet ${facet} $UMOUNT $@ $mntpt + fi # umount should block, but we should wait for unrelated obd's # like the MGS or MGC to also stop. @@ -1641,49 +2042,16 @@ stop() { if [[ $(facet_fstype $facet) == zfs ]]; then # export ZFS storage pool [ "$KEEP_ZPOOL" = "true" ] || export_zpool $facet + elif dm_flakey_supported $facet; then + local host=${facet}_HOST + local failover_host=${facet}failover_HOST + if [[ -n ${!failover_host} && ${!failover_host} != ${!host} ]]|| + $CLEANUP_DM_DEV || [[ $facet = fs* ]]; then + dm_cleanup_dev $facet + fi fi } -# save quota version (both administrative and operational quotas) -# add an additional parameter if mountpoint is ever different from $MOUNT -# -# XXX This function is kept for interoperability with old server (< 2.3.50), -# it should be removed whenever we drop the interoperability for such -# server. -quota_save_version() { - local fsname=${2:-$FSNAME} - local spec=$1 - local ver=$(tr -c -d "123" <<< $spec) - local type=$(tr -c -d "ug" <<< $spec) - - [ -n "$ver" -a "$ver" != "3" ] && error "wrong quota version specifier" - - [ -n "$type" ] && { $LFS quotacheck -$type $MOUNT || error "quotacheck has failed"; } - - do_facet mgs "lctl conf_param ${fsname}-MDT*.mdd.quota_type=$spec" - local varsvc - local osts=$(get_facets OST) - for ost in ${osts//,/ }; do - varsvc=${ost}_svc - do_facet mgs "lctl conf_param ${!varsvc}.ost.quota_type=$spec" - done -} - -# client could mount several lustre -# -# XXX This function is kept for interoperability with old server (< 2.3.50), -# it should be removed whenever we drop the interoperability for such -# server. -quota_type() { - local fsname=${1:-$FSNAME} - local rc=0 - do_facet $SINGLEMDS lctl get_param mdd.${fsname}-MDT*.quota_type || - rc=$? - do_nodes $(comma_list $(osts_nodes)) \ - lctl get_param obdfilter.${fsname}-OST*.quota_type || rc=$? - return $rc -} - # get mdt quota type mdt_quota_type() { local varsvc=${SINGLEMDS}_svc @@ -1702,12 +2070,24 @@ ost_quota_type() { # restore old quota type settings restore_quota() { if [ "$old_MDT_QUOTA_TYPE" ]; then - do_facet mgs $LCTL conf_param \ - $FSNAME.quota.mdt=$old_MDT_QUOTA_TYPE + if [[ $PERM_CMD = *"set_param -P"* ]]; then + do_facet mgs $PERM_CMD \ + osd-*.$FSNAME-MDT*.quota_slave.enable = \ + $old_MDT_QUOTA_TYPE + else + do_facet mgs $PERM_CMD \ + $FSNAME.quota.mdt=$old_MDT_QUOTA_TYPE + fi fi if [ "$old_OST_QUOTA_TYPE" ]; then - do_facet mgs $LCTL conf_param \ - $FSNAME.quota.ost=$old_OST_QUOTA_TYPE + if [[ $PERM_CMD = *"set_param -P"* ]]; then + do_facet mgs $PERM_CMD \ + osd-*.$FSNAME-OST*.quota_slave.enable = \ + $old_OST_QUOTA_TYPE + else + do_facet mgs $LCTL conf_param \ + $FSNAME.quota.ost=$old_OST_QUOTA_TYPE + fi fi } @@ -1760,10 +2140,17 @@ setup_quota(){ export old_MDT_QUOTA_TYPE=$mdt_qtype export old_OST_QUOTA_TYPE=$ost_qtype - do_facet mgs $LCTL conf_param $FSNAME.quota.mdt=$QUOTA_TYPE || - error "set mdt quota type failed" - do_facet mgs $LCTL conf_param $FSNAME.quota.ost=$QUOTA_TYPE || - error "set ost quota type failed" + if [[ $PERM_CMD = *"set_param -P"* ]]; then + do_facet mgs $PERM_CMD \ + osd-*.$FSNAME-MDT*.quota_slave.enable=$QUOTA_TYPE + do_facet mgs $PERM_CMD \ + osd-*.$FSNAME-OST*.quota_slave.enable=$QUOTA_TYPE + else + do_facet mgs $PERM_CMD $FSNAME.quota.mdt=$QUOTA_TYPE || + error "set mdt quota type failed" + do_facet mgs $PERM_CMD $FSNAME.quota.ost=$QUOTA_TYPE || + error "set ost quota type failed" + fi local quota_usrs=$QUOTA_USERS @@ -2159,13 +2546,23 @@ facets_up_on_host () { } shutdown_facet() { - local facet=$1 + local facet=$1 + local affected_facet + local affected_facets + + if [[ "$FAILURE_MODE" = HARD ]]; then + if [[ $(facet_fstype $facet) = ldiskfs ]] && + dm_flakey_supported $facet; then + affected_facets=$(affected_facets $facet) + for affected_facet in ${affected_facets//,/ }; do + unexport_dm_dev $affected_facet + done + fi - if [ "$FAILURE_MODE" = HARD ]; then - shutdown_node_hard $(facet_active_host $facet) - else - stop $facet - fi + shutdown_node_hard $(facet_active_host $facet) + else + stop $facet + fi } reboot_node() { @@ -2694,7 +3091,7 @@ wait_mds_ost_sync () { done # show which nodes are not finished. - cmd=$(echo $cmd | sed '/-n//') + cmd=$(echo $cmd | sed 's/-n//') do_nodes $list "$cmd" echo "$facet recovery node $i not done in $WAIT_TIMEOUT sec. $STATUS" return 1 @@ -2753,7 +3150,8 @@ wait_exit_ST () { local running # conf-sanity 31 takes a long time cleanup while [ $WAIT -lt 300 ]; do - running=$(do_facet ${facet} "lsmod | grep lnet > /dev/null && lctl dl | grep ' ST '") || true + running=$(do_facet ${facet} "lsmod | grep lnet > /dev/null && +lctl dl | grep ' ST ' || true") [ -z "${running}" ] && return 0 echo "waited $WAIT for${running}" [ $INTERVAL -lt 64 ] && INTERVAL=$((INTERVAL + INTERVAL)) @@ -2955,7 +3353,7 @@ replay_barrier() { # handled by stop() and mount_facet() separately, which are used # inside fail() and fail_abort(). # - do_facet $facet $LCTL --device ${!svc} readonly + set_dev_readonly $facet do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}" $LCTL mark "local REPLAY BARRIER on ${!svc}" } @@ -2966,7 +3364,7 @@ replay_barrier_nodf() { local svc=${facet}_svc echo Replay barrier on ${!svc} do_facet $facet $LCTL --device ${!svc} notransno - do_facet $facet $LCTL --device ${!svc} readonly + set_dev_readonly $facet do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}" $LCTL mark "local REPLAY BARRIER on ${!svc}" } @@ -2976,7 +3374,7 @@ replay_barrier_nosync() { local svc=${facet}_svc echo Replay barrier on ${!svc} do_facet $facet $LCTL --device ${!svc} notransno - do_facet $facet $LCTL --device ${!svc} readonly + set_dev_readonly $facet do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}" $LCTL mark "local REPLAY BARRIER on ${!svc}" } @@ -3014,7 +3412,9 @@ fail() { local clients=${CLIENTS:-$HOSTNAME} facet_failover $* || error "failover: $?" - wait_clients_import_state "$clients" "$facets" FULL + # to initiate all OSC idling connections + clients_up + wait_clients_import_state "$clients" "$facets" "\(FULL\|IDLE\)" clients_up || error "post-failover stat: $?" } @@ -3221,59 +3621,20 @@ facet_failover_host() { fi } -detect_active() { - local facet=$1 - [ "$CLIENTONLY" ] && echo $facet && return - - local failover=$(facet_failover_host $facet) - - # failover is not associated with all facet types: - # "AGT" facet type (remote HSM agents) does not - # have a failover. - [[ -z "$failover" ]] && echo $facet && return - - local host=$(facet_host $facet) - local dev=$(facet_device $facet) - - # ${facet}_svc can not be used here because of - # facet_active() is called before this var initialized - local svc=$(do_node $host $E2LABEL ${dev}) - - # active facet is ${facet}failover if device is mounted on failover - # on other cases active facet is $facet - [[ $dev = $(do_node $failover \ - lctl get_param -n *.$svc.mntdev 2>/dev/null) ]] && - echo ${facet}failover && return - - echo $facet -} - -init_active() { - local facet=$1 - - local active=$(detect_active $facet) - echo "${facet}active=$active" > $TMP/${facet}active -} - facet_active() { - local facet=$1 - local activevar=${facet}active + local facet=$1 + local activevar=${facet}active - # file is missing (nothing to store) if fail() is not - # executed during this test session yet; - # file content: - # ost1active=ost1failover - # ost1active=ost1 - # let's detect active facet based on current lustre state - if [ ! -f $TMP/${facet}active ] ; then - init_active $facet - fi - source $TMP/${facet}active + if [ -f $TMP/${facet}active ] ; then + source $TMP/${facet}active + fi - # is ${facet}active set somewhere else? - active=${!activevar} - [[ -z "$active" ]] && exit 1 - echo -n ${active} + active=${!activevar} + if [ -z "$active" ] ; then + echo -n ${facet} + else + echo -n ${active} + fi } facet_active_host() { @@ -3347,16 +3708,20 @@ do_node() { $myPDSH $HOST "$LCTL mark \"$@\"" > /dev/null 2>&1 || : fi - if [ "$myPDSH" = "rsh" ]; then -# we need this because rsh does not return exit code of an executed command - local command_status="$TMP/cs" - rsh $HOST ":> $command_status" - rsh $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; - cd $RPWD; LUSTRE=\"$RLUSTRE\" sh -c \"$@\") || - echo command failed >$command_status" - [ -n "$($myPDSH $HOST cat $command_status)" ] && return 1 || true - return 0 - fi + if [[ "$myPDSH" == "rsh" ]] || + [[ "$myPDSH" == *pdsh* && "$myPDSH" != *-S* ]]; then + # we need this because rsh and pdsh do not return + # exit code of an executed command + local command_status="$TMP/cs" + eval $myPDSH $HOST ":> $command_status" + eval $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests; + PATH=\$PATH:/sbin:/usr/sbin; + cd $RPWD; + LUSTRE=\"$RLUSTRE\" sh -c \"$@\") || + echo command failed >$command_status" + [[ -n "$($myPDSH $HOST cat $command_status)" ]] && return 1 || + return 0 + fi if $verbose ; then # print HOSTNAME for myPDSH="no_dsh" @@ -3533,6 +3898,8 @@ ostdevname() { case $fstype in ldiskfs ) + local dev=ost${num}_dev + [[ -n ${!dev} ]] && eval DEVPTR=${!dev} || #if $OSTDEVn isn't defined, default is $OSTDEVBASE + num eval DEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}};; zfs ) @@ -3579,6 +3946,8 @@ mdsdevname() { case $fstype in ldiskfs ) + local dev=mds${num}_dev + [[ -n ${!dev} ]] && eval DEVPTR=${!dev} || #if $MDSDEVn isn't defined, default is $MDSDEVBASE{n} eval DEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}};; zfs ) @@ -3621,9 +3990,10 @@ mgsdevname() { case $fstype in ldiskfs ) if [ $(facet_host mgs) = $(facet_host mds1) ] && - ( [ -z "$MGSDEV" ] || [ $MGSDEV = $(mdsdevname 1) ] ); then + ( [ -z "$MGSDEV" ] || [ $MGSDEV = $MDSDEV1 ] ); then DEVPTR=$(mdsdevname 1) else + [[ -n $mgs_dev ]] && DEVPTR=$mgs_dev || DEVPTR=$MGSDEV fi;; zfs ) @@ -3680,11 +4050,16 @@ mount_ldiskfs() { local dev=$(facet_device $facet) local mnt=${2:-$(facet_mntpt $facet)} local opts + local dm_dev=$dev - if ! do_facet $facet test -b $dev; then - opts="-o loop" + if dm_flakey_supported $facet; then + dm_dev=$(dm_create_dev $facet $dev) + [[ -n "$dm_dev" ]] || dm_dev=$dev fi - do_facet $facet mount -t ldiskfs $opts $dev $mnt + is_blkdev $facet $dm_dev || opts=$(csa_add "$opts" -o loop) + export_dm_dev $facet $dm_dev + + do_facet $facet mount -t ldiskfs $opts $dm_dev $mnt } unmount_ldiskfs() { @@ -3759,6 +4134,13 @@ unmount_fstype() { ## MountConf setup stopall() { + # make sure we are using the primary server, so test-framework will + # be able to clean up properly. + activemds=`facet_active mds1` + if [ $activemds != "mds1" ]; then + fail mds1 + fi + local clients=$CLIENTS [ -z $clients ] && clients=$(hostname) @@ -3803,8 +4185,8 @@ cleanupall() { nfs_client_mode && return cifs_client_mode && return - stopall $* cleanup_echo_devs + CLEANUP_DM_DEV=true stopall $* unload_modules cleanup_sk @@ -3859,15 +4241,15 @@ mkfs_opts() { if [ $type == MDS ]; then opts+=${MDSCAPA:+" --param-mdt.capa=$MDSCAPA"} - opts+=${STRIPE_BYTES:+" --param=lov.stripesize=$STRIPE_BYTES"} - opts+=${STRIPES_PER_OBJ:+" --param=lov.stripecount=$STRIPES_PER_OBJ"} + opts+=${DEF_STRIPE_SIZE:+" --param=lov.stripesize=$DEF_STRIPE_SIZE"} + opts+=${DEF_STRIPE_COUNT:+" --param=lov.stripecount=$DEF_STRIPE_COUNT"} opts+=${L_GETIDENTITY:+" --param=mdt.identity_upcall=$L_GETIDENTITY"} if [ $fstype == ldiskfs ]; then # Check for wide striping if [ $OSTCOUNT -gt 160 ]; then MDSJOURNALSIZE=${MDSJOURNALSIZE:-4096} - fs_mkfs_opts+="-O large_xattr" + fs_mkfs_opts+="-O ea_inode" fi var=${facet}_JRN @@ -4131,18 +4513,23 @@ mountmgs() { } mountmds() { + local num + local devname + local host + local varname for num in $(seq $MDSCOUNT); do - DEVNAME=$(mdsdevname $num) - start mds$num $DEVNAME $MDS_MOUNT_OPTS - - # We started mds, now we should set failover variables properly. - # Set mds${num}failover_HOST if unset (the default - # failnode). - local varname=mds${num}failover_HOST - if [ -z "${!varname}" ]; then - eval mds${num}failover_HOST=$(facet_host mds$num) - fi - + devname=$(mdsdevname $num) + start mds$num $devname $MDS_MOUNT_OPTS + + # We started mds$num, now we should set mds${num}_HOST + # and mds${num}failover_HOST variables properly if they + # are not set. + host=$(facet_host mds$num) + for varname in mds${num}_HOST mds${num}failover_HOST; do + if [[ -z "${!varname}" ]]; then + eval $varname=$host + fi + done if [ $IDENTITY_UPCALL != "default" ]; then switch_identity $num $IDENTITY_UPCALL fi @@ -4150,18 +4537,23 @@ mountmds() { } mountoss() { + local num + local devname + local host + local varname for num in $(seq $OSTCOUNT); do - DEVNAME=$(ostdevname $num) - start ost$num $DEVNAME $OST_MOUNT_OPTS - - # We started ost$num, now we should set ost${num}failover - # variable properly. Set ost${num}failover_HOST if it is not - # set (the default failnode). - varname=ost${num}failover_HOST - if [ -z "${!varname}" ]; then - eval ost${num}failover_HOST=$(facet_host ost${num}) - fi - + devname=$(ostdevname $num) + start ost$num $devname $OST_MOUNT_OPTS + + # We started ost$num, now we should set ost${num}_HOST + # and ost${num}failover_HOST variables properly if they + # are not set. + host=$(facet_host ost$num) + for varname in ost${num}_HOST ost${num}failover_HOST; do + if [[ -z "${!varname}" ]]; then + eval $varname=$host + fi + done done } @@ -4265,8 +4657,17 @@ setupall() { if $GSS_SK; then set_rule $FSNAME any cli2mdt $SK_FLAVOR set_rule $FSNAME any cli2ost $SK_FLAVOR - wait_flavor cli2mdt $SK_FLAVOR - wait_flavor cli2ost $SK_FLAVOR + if $SK_SKIPFIRST; then + export SK_SKIPFIRST=false + + sleep 30 + do_nodes $CLIENTS \ + "lctl set_param osc.*.idle_connect=1" + return + else + wait_flavor cli2mdt $SK_FLAVOR + wait_flavor cli2ost $SK_FLAVOR + fi else set_flavor_all $SEC fi @@ -4410,10 +4811,57 @@ set_conf_param_and_check() { error "check $PARAM failed!" } +set_persistent_param() { + local myfacet=$1 + local test_param=$2 + local param=$3 + local orig=$(do_facet $myfacet "$LCTL get_param -n $test_param") + + if [ $# -gt 3 ]; then + local final=$4 + else + local -i final + final=$((orig + 5)) + fi + + if [[ $PERM_CMD = *"set_param -P"* ]]; then + echo "Setting $test_param from $orig to $final" + do_facet mgs "$PERM_CMD $test_param='$final'" || + error "$PERM_CMD $test_param failed" + else + echo "Setting $param from $orig to $final" + do_facet mgs "$PERM_CMD $param='$final'" || + error "$PERM_CMD $param failed" + fi +} + +set_persistent_param_and_check() { + local myfacet=$1 + local test_param=$2 + local param=$3 + local orig=$(do_facet $myfacet "$LCTL get_param -n $test_param") + + if [ $# -gt 3 ]; then + local final=$4 + else + local -i final + final=$((orig + 5)) + fi + + set_persistent_param $myfacet $test_param $param "$final" + + wait_update_facet $myfacet "$LCTL get_param -n $test_param" "$final" || + error "check $param failed!" +} + init_param_vars () { TIMEOUT=$(lctl get_param -n timeout) TIMEOUT=${TIMEOUT:-20} + if [ -n $arg1 ]; then + [ "$arg1" = "server_only" ] && return + fi + remote_mds_nodsh && log "Using TIMEOUT=$TIMEOUT" && return 0 TIMEOUT=$(do_facet $SINGLEMDS "lctl get_param -n timeout") @@ -4421,6 +4869,7 @@ init_param_vars () { osc_ensure_active $SINGLEMDS $TIMEOUT osc_ensure_active client $TIMEOUT + $LCTL set_param osc.*.idle_timeout=debug if [ -n "$(lctl get_param -n mdc.*.connect_flags|grep jobstats)" ]; then local current_jobid_var=$($LCTL get_param -n jobid_var) @@ -4428,11 +4877,10 @@ init_param_vars () { if [ $JOBID_VAR = "existing" ]; then echo "keeping jobstats as $current_jobid_var" elif [ $current_jobid_var != $JOBID_VAR ]; then - echo "seting jobstats to $JOBID_VAR" + echo "setting jobstats to $JOBID_VAR" - set_conf_param_and_check client \ - "$LCTL get_param -n jobid_var" \ - "$FSNAME.sys.jobid_var" $JOBID_VAR + set_persistent_param_and_check client \ + "jobid_var" "$FSNAME.sys.jobid_var" $JOBID_VAR fi else echo "jobstats not supported by server" @@ -4566,7 +5014,7 @@ check_and_setup_lustre() { # 1. # both MOUNT and MOUNT2 are not mounted if ! is_mounted $MOUNT && ! is_mounted $MOUNT2; then - [ "$REFORMAT" = "yes" ] && formatall + [ "$REFORMAT" = "yes" ] && CLEANUP_DM_DEV=true formatall # setupall mounts both MOUNT and MOUNT2 (if MOUNT_2 is set) setupall is_mounted $MOUNT || error "NAME=$NAME not mounted" @@ -4630,7 +5078,6 @@ check_and_setup_lustre() { fi fi - init_gss if $GSS_SK; then set_flavor_all null elif $GSS; then @@ -4736,7 +5183,7 @@ run_e2fsck() { if [ -n "$(grep "DNE mode isn't supported" $log)" ]; then rm -f $log if [ $MDSCOUNT -gt 1 ]; then - skip "DNE mode isn't supported!" + skip_noexit "DNE mode isn't supported!" cleanupall exit_status else @@ -4880,7 +5327,7 @@ check_and_cleanup_lustre() { cleanup_mount $MOUNT2 fi - if [ "$I_MOUNTED" = "yes" ]; then + if [[ "$I_MOUNTED" = "yes" ]] && ! $AUSTER_CLEANUP; then cleanupall -f || error "cleanup failed" unset I_MOUNTED fi @@ -5419,7 +5866,7 @@ skip_env () { $FAIL_ON_SKIP_ENV && error false $@ || skip $@ } -skip() { +skip_noexit() { echo log " SKIP: $TESTSUITE $TESTNAME $@" @@ -5434,8 +5881,13 @@ skip() { echo "$TESTSUITE: SKIP: $TESTNAME $@" >> $TESTSUITELOG || true } +skip() { + skip_noexit $@ + exit 0 +} + build_test_filter() { - EXCEPT="$EXCEPT $(testslist_filter)" + EXCEPT="$EXCEPT $(testslist_filter)" for O in $ONLY; do if [[ $O = [0-9]*-[0-9]* ]]; then @@ -5509,32 +5961,32 @@ run_test() { ALWAYS_SKIPPED="y" testname=EXCEPT_$1 if [ ${!testname}x != x ]; then - TESTNAME=test_$1 skip "skipping excluded test $1" + TESTNAME=test_$1 skip_noexit "skipping excluded test $1" return 0 fi testname=EXCEPT_$base if [ ${!testname}x != x ]; then - TESTNAME=test_$1 skip "skipping excluded test $1 (base $base)" + TESTNAME=test_$1 skip_noexit "skipping excluded test $1 (base $base)" return 0 fi testname=EXCEPT_ALWAYS_$1 if [ ${!testname}x != x ]; then - TESTNAME=test_$1 skip "skipping ALWAYS excluded test $1" + TESTNAME=test_$1 skip_noexit "skipping ALWAYS excluded test $1" return 0 fi testname=EXCEPT_ALWAYS_$base if [ ${!testname}x != x ]; then - TESTNAME=test_$1 skip "skipping ALWAYS excluded test $1 (base $base)" + TESTNAME=test_$1 skip_noexit "skipping ALWAYS excluded test $1 (base $base)" return 0 fi testname=EXCEPT_SLOW_$1 if [ ${!testname}x != x ]; then - TESTNAME=test_$1 skip "skipping SLOW test $1" + TESTNAME=test_$1 skip_noexit "skipping SLOW test $1" return 0 fi testname=EXCEPT_SLOW_$base if [ ${!testname}x != x ]; then - TESTNAME=test_$1 skip "skipping SLOW test $1 (base $base)" + TESTNAME=test_$1 skip_noexit "skipping SLOW test $1 (base $base)" return 0 fi @@ -5754,6 +6206,7 @@ check_grant() { # sync all the data and make sure no pending data on server do_nodes $clients sync + clients_up # initiate all idling connections # get client grant client_grant=$(do_nodes $clients \ @@ -6328,7 +6781,7 @@ calc_sum () { } calc_osc_kbytes () { - df $MOUNT > /dev/null + $LFS df $MOUNT > /dev/null $LCTL get_param -n osc.*[oO][sS][cC][-_][0-9a-f]*.$1 | calc_sum } @@ -6358,7 +6811,7 @@ restore_lustre_params() { local val while IFS=" =" read facet name val; do - do_facet $facet "$LCTL set_param -n $name $val" + do_facet $facet "$LCTL set_param -n $name=$val" done } @@ -6466,7 +6919,7 @@ _wait_import_state () { local i=0 CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2 | uniq) - while [ "${CONN_STATE}" != "${expected}" ]; do + while ! echo "${CONN_STATE}" | egrep -q "^${expected}\$" ; do if [ "${expected}" == "DISCONN" ]; then # for disconn we can check after proc entry is removed [ "x${CONN_STATE}" == "x" ] && return 0 @@ -6611,6 +7064,10 @@ wait_osc_import_state() { fi } +wait_osc_import_ready() { + wait_osc_import_state $1 $2 "\(FULL\|IDLE\)" +} + _wait_mgc_import_state() { local facet=$1 local expected=$2 @@ -6673,7 +7130,7 @@ wait_dne_interconnect() { if [ $MDSCOUNT -gt 1 ]; then for num in $(seq $MDSCOUNT); do - wait_osc_import_state mds mds$num FULL + wait_osc_import_ready mds mds$num done fi } @@ -6696,7 +7153,7 @@ do_rpc_nodes () { local LIBPATH="/usr/lib/lustre/tests:/usr/lib64/lustre/tests:" local TESTPATH="$RLUSTRE/tests:" local RPATH="PATH=${TESTPATH}${LIBPATH}${PATH}:/sbin:/bin:/usr/sbin:" - do_nodesv $list "${RPATH} NAME=${NAME} sh rpc.sh $@ " + do_nodesv $list "${RPATH} NAME=${NAME} bash rpc.sh $@ " } wait_clients_import_state () { @@ -6726,7 +7183,7 @@ wait_clients_import_state () { local params=$(expand_list $params $proc_path) done - if ! do_rpc_nodes "$list" wait_import_state_mount $expected $params; + if ! do_rpc_nodes "$list" wait_import_state_mount "$expected" $params; then error "import is not in ${expected} state" return 1 @@ -6743,6 +7200,7 @@ wait_osp_active() { # wait until all MDTs are in the expected state for ((num = 1; num <= $MDSCOUNT; num++)); do local mdtosp=$(get_mdtosc_proc_path mds${num} ${tgt_name}) + local wait=0 local mproc if [ $facet = "mds" ]; then @@ -6757,7 +7215,6 @@ wait_osp_active() { sleep 5 local result=$(do_facet mds${num} "$LCTL get_param -n $mproc") local max=30 - local wait=0 [ ${PIPESTATUS[0]} = 0 ] || error "Can't read $mproc" if [ $result -eq $expected ]; then @@ -6818,7 +7275,7 @@ create_pool() { local fsname=${1%%.*} local poolname=${1##$fsname.} - trap "destroy_test_pools $fsname" EXIT + stack_trap "destroy_test_pools $fsname" EXIT do_facet mgs lctl pool_new $1 local RC=$? # get param should return err unless pool is created @@ -7186,6 +7643,8 @@ flvr_cnt_cli2ost() local clients=${CLIENTS:-$HOSTNAME} for c in ${clients//,/ }; do + # reconnect if idle + do_node $c lctl set_param osc.*.idle_connect=1 >/dev/null 2>&1 local output=$(do_node $c lctl get_param -n \ osc.*OST*-osc-[^M][^D][^T]*.$PROC_CLI 2>/dev/null) local tmpcnt=$(count_flvr "$output" $flavor) @@ -7623,15 +8082,20 @@ run_sgpdd () { # returns the canonical name for an ldiskfs device ldiskfs_canon() { - local dev="$1" - local facet="$2" - - do_facet $facet "dv=\\\$(lctl get_param -n $dev); -if foo=\\\$(lvdisplay -c \\\$dv 2>/dev/null); then - echo dm-\\\${foo##*:}; -else - echo \\\$(basename \\\$dv); -fi;" + local dev="$1" + local facet="$2" + + do_facet $facet "dv=\\\$($LCTL get_param -n $dev); + if foo=\\\$(lvdisplay -c \\\$dv 2>/dev/null); then + echo dm-\\\${foo##*:}; + else + name=\\\$(basename \\\$dv); + if [[ \\\$name = *flakey* ]]; then + name=\\\$(lsblk -o NAME,KNAME | + awk /\\\$name/'{print \\\$NF}'); + fi; + echo \\\$name; + fi;" } is_sanity_benchmark() { @@ -7665,13 +8129,15 @@ get_obd_size() { # # Get the page size (bytes) on a given facet node. +# The local client page_size is directly available in PAGE_SIZE. # get_page_size() { local facet=$1 - local size=$(getconf PAGE_SIZE 2>/dev/null) + local page_size=$(getconf PAGE_SIZE 2>/dev/null) - [ -z "$CLIENTONLY" ] && size=$(do_facet $facet getconf PAGE_SIZE) - echo -n ${size:-4096} + [ -z "$CLIENTONLY" -a "$facet" != "client" ] && + page_size=$(do_facet $facet getconf PAGE_SIZE) + echo -n ${page_size:-4096} } # @@ -7698,7 +8164,9 @@ get_block_size() { echo -n ${size:-0} } -# Check whether the "large_xattr" feature is enabled or not. +# Check whether the "ea_inode" feature is enabled or not, to allow +# ldiskfs xattrs over one block in size. Allow both the historical +# Lustre feature name (large_xattr) and the upstream name (ea_inode). large_xattr_enabled() { [[ $(facet_fstype $SINGLEMDS) == zfs ]] && return 0 @@ -8374,8 +8842,7 @@ lss_gen_conf() if ! combined_mgs_mds ; then [ $(facet_fstype mgs) != zfs ] && - skip "Lustre snapshot 1 only works for ZFS backend" && - exit 0 + skip "Lustre snapshot 1 only works for ZFS backend" local host=$(facet_active_host mgs) local dir=$(dirname $(facet_vdevice mgs)) @@ -8389,8 +8856,7 @@ lss_gen_conf() for num in `seq $MDSCOUNT`; do [ $(facet_fstype mds$num) != zfs ] && - skip "Lustre snapshot 1 only works for ZFS backend" && - exit 0 + skip "Lustre snapshot 1 only works for ZFS backend" lss_gen_conf_one mds$num MDT $((num - 1)) || lss_err "generate lss conf (mds$num)" @@ -8398,8 +8864,7 @@ lss_gen_conf() for num in `seq $OSTCOUNT`; do [ $(facet_fstype ost$num) != zfs ] && - skip "Lustre snapshot 1 only works for ZFS backend" && - exit 0 + skip "Lustre snapshot 1 only works for ZFS backend" lss_gen_conf_one ost$num OST $((num - 1)) || lss_err "generate lss conf (ost$num)" @@ -8562,7 +9027,327 @@ check_clients_full() { done } -# restore the layout saved by save_layout(). Only work with directories +#Changelogs +__changelog_deregister() { + local facet=$1 + local mdt="$(facet_svc $facet)" + local cl_user=$2 + local rc=0 + + # skip cleanup if no user registered for this MDT + [ -z "$cl_user" ] && echo "$mdt: no changelog user" && return 0 + # user is no longer registered, skip cleanup + changelog_users "$facet" | grep -q "$cl_user" || + { echo "$mdt: changelog user '$cl_user' not found"; return 0; } + + # From this point, if any operation fails, it is an error + __changelog_clear $facet $cl_user 0 || + error_noexit "$mdt: changelog_clear $cl_user 0 fail: $rc" + do_facet $facet $LCTL --device $mdt changelog_deregister $cl_user || + error_noexit "$mdt: changelog_deregister '$cl_user' fail: $rc" +} + +declare -Ax CL_USERS +changelog_register() { + for M in $(seq $MDSCOUNT); do + local facet=mds$M + local mdt="$(facet_svc $facet)" + local cl_mask + + cl_mask=$(do_facet $facet $LCTL get_param \ + mdd.${mdt}.changelog_mask -n) + stack_trap "do_facet $facet $LCTL \ + set_param mdd.$mdt.changelog_mask=\'$cl_mask\' -n" EXIT + do_facet $facet $LCTL set_param mdd.$mdt.changelog_mask=+hsm || + error "$mdt: changelog_mask=+hsm failed: $?" + + local cl_user + cl_user=$(do_facet $facet \ + $LCTL --device $mdt changelog_register -n) || + error "$mdt: register changelog user failed: $?" + stack_trap "__changelog_deregister $facet $cl_user" EXIT + + stack_trap "CL_USERS[$facet]='${CL_USERS[$facet]}'" EXIT + # Bash does not support nested arrays, but the format of a + # cl_user is constrained enough to use whitespaces as separators + CL_USERS[$facet]+="$cl_user " + done + echo "Registered $MDSCOUNT changelog users: '${CL_USERS[@]% }'" +} + +changelog_deregister() { + local cl_user + # bash assoc arrays do not guarantee to list keys in created order + # so reorder to get same order than in changelog_register() + local cl_facets=$(echo "${!CL_USERS[@]}" | tr " " "\n" | sort | + tr "\n" " ") + + for facet in $cl_facets; do + for cl_user in ${CL_USERS[$facet]}; do + __changelog_deregister $facet $cl_user || return $? + done + unset CL_USERS[$facet] + done +} + +changelog_users() { + local facet=$1 + local service=$(facet_svc $facet) + + do_facet $facet $LCTL get_param -n mdd.$service.changelog_users +} + +changelog_user_rec() { + local facet=$1 + local cl_user=$2 + local service=$(facet_svc $facet) + + changelog_users $facet | awk '$1 == "'$cl_user'" { print $2 }' +} + +changelog_chmask() { + local mask=$1 + + do_nodes $(comma_list $(mdts_nodes)) \ + $LCTL set_param mdd.*.changelog_mask="$mask" +} + +# usage: __changelog_clear FACET CL_USER [+]INDEX +__changelog_clear() +{ + local facet=$1 + local mdt="$(facet_svc $facet)" + local cl_user=$2 + local -i rec + + case "$3" in + +*) + # Remove the leading '+' + rec=${3:1} + rec+=$(changelog_user_rec $facet $cl_user) + ;; + *) + rec=$3 + ;; + esac + + if [ $rec -eq 0 ]; then + echo "$mdt: clear the changelog for $cl_user of all records" + else + echo "$mdt: clear the changelog for $cl_user to record #$rec" + fi + $LFS changelog_clear $mdt $cl_user $rec +} + +# usage: changelog_clear [+]INDEX +# +# If INDEX is prefixed with '+', increment every changelog user's record index +# by INDEX. Otherwise, clear the changelog up to INDEX for every changelog +# users. +changelog_clear() { + local rc + # bash assoc arrays do not guarantee to list keys in created order + # so reorder to get same order than in changelog_register() + local cl_facets=$(echo "${!CL_USERS[@]}" | tr " " "\n" | sort | + tr "\n" " ") + + for facet in $cl_facets; do + for cl_user in ${CL_USERS[$facet]}; do + __changelog_clear $facet $cl_user $1 || rc=${rc:-$?} + done + done + + return ${rc:-0} +} + +changelog_dump() { + for M in $(seq $MDSCOUNT); do + local facet=mds$M + local mdt="$(facet_svc $facet)" + + $LFS changelog $mdt | sed -e 's/^/'$mdt'./' + done +} + +changelog_extract_field() { + local cltype=$1 + local file=$2 + local identifier=$3 + + changelog_dump | gawk "/$cltype.*$file$/ { + print gensub(/^.* "$identifier'(\[[^\]]*\]).*$/,"\\1",1)}' | + tail -1 +} + +# Prints a changelog record produced by "lfs changelog" as an associative array +# +# Example: +# $> changelog2array 16 01CREAT 10:28:46.968438800 2018.03.09 0x0 \ +# t=[0x200000401:0x10:0x0] j=touch.501 ef=0xf u=501:501 \ +# nid=0@lo p=[0x200000007:0x1:0x0] blob +# ([index]='16' [type]='CREAT' [time]='10:28:46.968438800' +# [date]='2018.03.09' [flags]=0x0 ['target-fid']='[0x200000401:0x10:0x0]' +# ['jobid']='touch.501' ['extra-flags']='0x0f' [uid]='0' ['gid']='0' +# ['nid']='0@lo' ['parent-fid']='[0x200000007:0x1:0x0]') +# +# Note that the changelog record is not quoted +# Also note that the line breaks in the output were only added for readability +# +# Typically, you want to eval the output of the command to fill an actual +# associative array, like this: +# $> eval declare -A changelog=$(changelog2array $entry) +# +# It can then be accessed like any bash associative array: +# $> echo "${changelog[index]}" "${changelog[type]}" "${changelog[flags]}" +# 16 CREAT 0x0 +# $> echo "${changelog[uid]}":"${changelog[gid]}" +# 501:501 +# +changelog2array() +{ + # Start the array + printf '(' + + # A changelog, as printed by "lfs changelog" typically looks like this: + #