export SK_S2SNMCLI=${SK_S2SNMCLI:-TestFrameNMCli}
export IDENTITY_UPCALL=default
export QUOTA_AUTO=1
+export FLAKEY=${FLAKEY:-true}
# specify environment variable containing batch job name for server statistics
export JOBID_VAR=${JOBID_VAR:-"procname_uid"} # or "existing" or "disable"
export RPC_MODE=${RPC_MODE:-false}
export DO_CLEANUP=${DO_CLEANUP:-true}
export KEEP_ZPOOL=${KEEP_ZPOOL:-false}
+ export CLEANUP_DM_DEV=false
export MKE2FS=$MKE2FS
if [ -z "$MKE2FS" ]; then
export SGPDDSURVEY=${SGPDDSURVEY:-"$LUSTRE/../lustre-iokit/sgpdd-survey/sgpdd-survey")}
[ ! -f "$SGPDDSURVEY" ] && export SGPDDSURVEY=$(which sgpdd-survey)
export MCREATE=${MCREATE:-mcreate}
+ export MULTIOP=${MULTIOP:-multiop}
# Ubuntu, at least, has a truncate command in /usr/bin
# so fully path our truncate command.
export TRUNCATE=${TRUNCATE:-$LUSTRE/tests/truncate}
export LUSTRE_RMMOD=${LUSTRE_RMMOD:-$LUSTRE/scripts/lustre_rmmod}
[ ! -f "$LUSTRE_RMMOD" ] &&
export LUSTRE_RMMOD=$(which lustre_rmmod 2> /dev/null)
+ export LUSTRE_ROUTES_CONVERSION=${LUSTRE_ROUTES_CONVERSION:-$LUSTRE/scripts/lustre_routes_conversion}
+ [ ! -f "$LUSTRE_ROUTES_CONVERSION" ] &&
+ export LUSTRE_ROUTES_CONVERSION=$(which lustre_routes_conversion 2> /dev/null)
export LFS_MIGRATE=${LFS_MIGRATE:-$LUSTRE/scripts/lfs_migrate}
[ ! -f "$LFS_MIGRATE" ] &&
export LFS_MIGRATE=$(which lfs_migrate 2> /dev/null)
export LDEV=${LDEV:-"$LUSTRE/scripts/ldev"}
[ ! -f "$LDEV" ] && export LDEV=$(which ldev 2> /dev/null)
+ export DMSETUP=${DMSETUP:-dmsetup}
+ export DM_DEV_PATH=${DM_DEV_PATH:-/dev/mapper}
+ export LOSETUP=${LOSETUP:-losetup}
+
if [ "$ACCEPTOR_PORT" ]; then
export PORT_OPT="--port $ACCEPTOR_PORT"
fi
export RLUSTRE=${RLUSTRE:-$LUSTRE}
export RPWD=${RPWD:-$PWD}
export I_MOUNTED=${I_MOUNTED:-"no"}
+ export AUSTER_CLEANUP=${AUSTER_CLEANUP:-false}
if [ ! -f /lib/modules/$(uname -r)/kernel/fs/lustre/mdt.ko -a \
! -f /lib/modules/$(uname -r)/updates/kernel/fs/lustre/mdt.ko -a \
! -f /lib/modules/$(uname -r)/extra/kernel/fs/lustre/mdt.ko -a \
LNETLND="socklnd/ksocklnd"
esac
fi
- load_module ../lnet/klnds/$LNETLND
- load_module obdclass/obdclass
- load_module ptlrpc/ptlrpc
- load_module ptlrpc/gss/ptlrpc_gss
- load_module fld/fld
- load_module fid/fid
- load_module lmv/lmv
- load_module mdc/mdc
- load_module osc/osc
- load_module lov/lov
- load_module mgc/mgc
- load_module obdecho/obdecho
+ load_module ../lnet/klnds/$LNETLND
+ load_module obdclass/obdclass
+ load_module ptlrpc/ptlrpc
+ load_module ptlrpc/gss/ptlrpc_gss
+ load_module fld/fld
+ load_module fid/fid
+ load_module lmv/lmv
+ load_module osc/osc
+ load_module mdc/mdc
+ load_module lov/lov
+ load_module mgc/mgc
+ load_module obdecho/obdecho
if ! client_only; then
SYMLIST=/proc/kallsyms
grep -q crc16 $SYMLIST ||
fs_log_size() {
local facet=${1:-$SINGLEMDS}
- local fstype=$(facet_fstype $facet)
local size=0
- case $fstype in
+
+ case $(facet_fstype $facet) in
ldiskfs) size=50;; # largest seen is 44, leave some headroom
- zfs) size=512;; # largest seen is 512
+ # grant_block_size is in bytes, allow at least 2x max blocksize
+ zfs) size=$(lctl get_param osc.$FSNAME*.import |
+ awk '/grant_block_size:/ {print $2/512; exit;}')
+ ;;
esac
echo -n $size
import_zpool() {
local facet=$1
shift
- local opts=${@:-"-o cachefile=none"}
+ local opts=${@:-"-o cachefile=none -o failmode=panic"}
local poolname
poolname=$(zpool_name $facet)
[ $RC -eq 0 ] && continue
if [ "$TESTSUITE.$TESTNAME" = "replay-dual.test_0a" ]; then
- skip "Restart of $facet failed!." && touch $LU482_FAILED
+ skip_noexit "Restart of $facet failed!." &&
+ touch $LU482_FAILED
else
error "Restart of $facet failed!"
fi
echo -n "$opts"
}
+#
+# Associate loop device with a given regular file.
+# Return the loop device.
+#
+setup_loop_device() {
+ local facet=$1
+ local file=$2
+
+ do_facet $facet "loop_dev=\\\$($LOSETUP -j $file | cut -d : -f 1);
+ if [[ -z \\\$loop_dev ]]; then
+ loop_dev=\\\$($LOSETUP -f);
+ $LOSETUP \\\$loop_dev $file || loop_dev=;
+ fi;
+ echo -n \\\$loop_dev"
+}
+
+#
+# Detach a loop device.
+#
+cleanup_loop_device() {
+ local facet=$1
+ local loop_dev=$2
+
+ do_facet $facet "! $LOSETUP $loop_dev >/dev/null 2>&1 ||
+ $LOSETUP -d $loop_dev"
+}
+
+#
+# Check if a given device is a block device.
+#
+is_blkdev() {
+ local facet=$1
+ local dev=$2
+ local size=${3:-""}
+
+ [[ -n "$dev" ]] || return 1
+ do_facet $facet "test -b $dev" || return 1
+ if [[ -n "$size" ]]; then
+ local in=$(do_facet $facet "dd if=$dev of=/dev/null bs=1k \
+ count=1 skip=$size 2>&1" |
+ awk '($3 == "in") { print $1 }')
+ [[ "$in" = "1+0" ]] || return 1
+ fi
+}
+
+#
+# Check if a given device is a device-mapper device.
+#
+is_dm_dev() {
+ local facet=$1
+ local dev=$2
+
+ [[ -n "$dev" ]] || return 1
+ do_facet $facet "$DMSETUP status $dev >/dev/null 2>&1"
+}
+
+#
+# Check if a given device is a device-mapper flakey device.
+#
+is_dm_flakey_dev() {
+ local facet=$1
+ local dev=$2
+ local type
+
+ [[ -n "$dev" ]] || return 1
+
+ type=$(do_facet $facet "$DMSETUP status $dev 2>&1" |
+ awk '{print $3}')
+ [[ $type = flakey ]] && return 0 || return 1
+}
+
+#
+# Check if device-mapper flakey device is supported by the kernel
+# of $facet node or not.
+#
+dm_flakey_supported() {
+ local facet=$1
+
+ $FLAKEY || return 1
+ do_facet $facet "modprobe dm-flakey;
+ $DMSETUP targets | grep -q flakey" &> /dev/null
+}
+
+#
+# Get the device-mapper flakey device name of a given facet.
+#
+dm_facet_devname() {
+ local facet=$1
+ [[ $facet = mgs ]] && combined_mgs_mds && facet=mds1
+
+ echo -n ${facet}_flakey
+}
+
+#
+# Get the device-mapper flakey device of a given facet.
+# A device created by dmsetup will appear as /dev/mapper/<device-name>.
+#
+dm_facet_devpath() {
+ local facet=$1
+
+ echo -n $DM_DEV_PATH/$(dm_facet_devname $facet)
+}
+
+#
+# Set a device-mapper device with a new table.
+#
+# The table has the following format:
+# <logical_start_sector> <num_sectors> <target_type> <target_args>
+#
+# flakey <target_args> includes:
+# <destination_device> <offset> <up_interval> <down_interval> \
+# [<num_features> [<feature_arguments>]]
+#
+# linear <target_args> includes:
+# <destination_device> <start_sector>
+#
+dm_set_dev_table() {
+ local facet=$1
+ local dm_dev=$2
+ local target_type=$3
+ local num_sectors
+ local real_dev
+ local tmp
+ local table
+
+ read tmp num_sectors tmp real_dev tmp \
+ <<< $(do_facet $facet "$DMSETUP table $dm_dev")
+
+ case $target_type in
+ flakey)
+ table="0 $num_sectors flakey $real_dev 0 0 1800 1 drop_writes"
+ ;;
+ linear)
+ table="0 $num_sectors linear $real_dev 0"
+ ;;
+ *) error "invalid target type $target_type" ;;
+ esac
+
+ do_facet $facet "$DMSETUP suspend --nolockfs --noflush $dm_dev" ||
+ error "failed to suspend $dm_dev"
+ do_facet $facet "$DMSETUP load $dm_dev --table \\\"$table\\\"" ||
+ error "failed to load $target_type table into $dm_dev"
+ do_facet $facet "$DMSETUP resume $dm_dev" ||
+ error "failed to resume $dm_dev"
+}
+
+#
+# Set a device-mapper flakey device as "read-only" by using the "drop_writes"
+# feature parameter.
+#
+# drop_writes:
+# All write I/O is silently ignored.
+# Read I/O is handled correctly.
+#
+dm_set_dev_readonly() {
+ local facet=$1
+ local dm_dev=${2:-$(dm_facet_devpath $facet)}
+
+ dm_set_dev_table $facet $dm_dev flakey
+}
+
+#
+# Set a device-mapper device to traditional linear mapping mode.
+#
+dm_clear_dev_readonly() {
+ local facet=$1
+ local dm_dev=${2:-$(dm_facet_devpath $facet)}
+
+ dm_set_dev_table $facet $dm_dev linear
+}
+
+#
+# Set the device of a given facet as "read-only".
+#
+set_dev_readonly() {
+ local facet=$1
+ local svc=${facet}_svc
+
+ if [[ $(facet_fstype $facet) = zfs ]] ||
+ ! dm_flakey_supported $facet; then
+ do_facet $facet $LCTL --device ${!svc} readonly
+ else
+ dm_set_dev_readonly $facet
+ fi
+}
+
+#
+# Get size in 512-byte sectors (BLKGETSIZE64 / 512) of a given device.
+#
+get_num_sectors() {
+ local facet=$1
+ local dev=$2
+ local num_sectors
+
+ num_sectors=$(do_facet $facet "blockdev --getsz $dev 2>/dev/null")
+ [[ ${PIPESTATUS[0]} = 0 && -n "$num_sectors" ]] || num_sectors=0
+ echo -n $num_sectors
+}
+
+#
+# Create a device-mapper device with a given block device or regular file (will
+# be associated with loop device).
+# Return the full path of the device-mapper device.
+#
+dm_create_dev() {
+ local facet=$1
+ local real_dev=$2 # destination device
+ local dm_dev_name=${3:-$(dm_facet_devname $facet)} # device name
+ local dm_dev=$DM_DEV_PATH/$dm_dev_name # device-mapper device
+
+ # check if the device-mapper device to be created already exists
+ if is_dm_dev $facet $dm_dev; then
+ # if the existing device was set to "read-only", then clear it
+ ! is_dm_flakey_dev $facet $dm_dev ||
+ dm_clear_dev_readonly $facet $dm_dev
+
+ echo -n $dm_dev
+ return 0
+ fi
+
+ # check if the destination device is a block device, and if not,
+ # associate it with a loop device
+ is_blkdev $facet $real_dev ||
+ real_dev=$(setup_loop_device $facet $real_dev)
+ [[ -n "$real_dev" ]] || { echo -n $real_dev; return 2; }
+
+ # now create the device-mapper device
+ local num_sectors=$(get_num_sectors $facet $real_dev)
+ local table="0 $num_sectors linear $real_dev 0"
+ local rc=0
+
+ do_facet $facet "$DMSETUP create $dm_dev_name --table \\\"$table\\\"" ||
+ { rc=${PIPESTATUS[0]}; dm_dev=; }
+ do_facet $facet "$DMSETUP mknodes >/dev/null 2>&1"
+
+ echo -n $dm_dev
+ return $rc
+}
+
+#
+# Map the facet name to its device variable name.
+#
+facet_device_alias() {
+ local facet=$1
+ local dev_alias=$facet
+
+ case $facet in
+ fs2mds) dev_alias=mds1_2 ;;
+ fs2ost) dev_alias=ost1_2 ;;
+ fs3ost) dev_alias=ost2_2 ;;
+ *) ;;
+ esac
+
+ echo -n $dev_alias
+}
+
+#
+# Save the original value of the facet device and export the new value.
+#
+export_dm_dev() {
+ local facet=$1
+ local dm_dev=$2
+
+ local active_facet=$(facet_active $facet)
+ local dev_alias=$(facet_device_alias $active_facet)
+ local dev_name=${dev_alias}_dev
+ local dev=${!dev_name}
+
+ if [[ $active_facet = $facet ]]; then
+ local failover_dev=${dev_alias}failover_dev
+ if [[ ${!failover_dev} = $dev ]]; then
+ eval export ${failover_dev}_saved=$dev
+ eval export ${failover_dev}=$dm_dev
+ fi
+ else
+ dev_alias=$(facet_device_alias $facet)
+ local facet_dev=${dev_alias}_dev
+ if [[ ${!facet_dev} = $dev ]]; then
+ eval export ${facet_dev}_saved=$dev
+ eval export ${facet_dev}=$dm_dev
+ fi
+ fi
+
+ eval export ${dev_name}_saved=$dev
+ eval export ${dev_name}=$dm_dev
+}
+
+#
+# Restore the saved value of the facet device.
+#
+unexport_dm_dev() {
+ local facet=$1
+
+ [[ $facet = mgs ]] && combined_mgs_mds && facet=mds1
+ local dev_alias=$(facet_device_alias $facet)
+
+ local saved_dev=${dev_alias}_dev_saved
+ [[ -z ${!saved_dev} ]] ||
+ eval export ${dev_alias}_dev=${!saved_dev}
+
+ saved_dev=${dev_alias}failover_dev_saved
+ [[ -z ${!saved_dev} ]] ||
+ eval export ${dev_alias}failover_dev=${!saved_dev}
+}
+
+#
+# Remove a device-mapper device.
+# If the destination device is a loop device, then also detach it.
+#
+dm_cleanup_dev() {
+ local facet=$1
+ local dm_dev=${2:-$(dm_facet_devpath $facet)}
+ local major
+ local minor
+
+ is_dm_dev $facet $dm_dev || return 0
+
+ read major minor <<< $(do_facet $facet "$DMSETUP table $dm_dev" |
+ awk '{ print $4 }' | awk -F: '{ print $1" "$2 }')
+
+ do_facet $facet "$DMSETUP remove $dm_dev"
+ do_facet $facet "$DMSETUP mknodes >/dev/null 2>&1"
+
+ unexport_dm_dev $facet
+
+ # detach a loop device
+ [[ $major -ne 7 ]] || cleanup_loop_device $facet /dev/loop$minor
+
+ # unload dm-flakey module
+ do_facet $facet "modprobe -r dm-flakey" || true
+}
+
mount_facet() {
local facet=$1
shift
- local dev=$(facet_active $facet)_dev
+ local active_facet=$(facet_active $facet)
+ local dev_alias=$(facet_device_alias $active_facet)
+ local dev=${dev_alias}_dev
local opt=${facet}_opt
local mntpt=$(facet_mntpt $facet)
local opts="${!opt} $@"
local fstype=$(facet_fstype $facet)
local devicelabel
+ local dm_dev=${!dev}
module_loaded lustre || load_modules
- if [ $(facet_fstype $facet) == ldiskfs ] &&
- ! do_facet $facet test -b ${!dev}; then
- opts=$(csa_add "$opts" -o loop)
- fi
+ case $fstype in
+ ldiskfs)
+ if dm_flakey_supported $facet; then
+ dm_dev=$(dm_create_dev $facet ${!dev})
+ [[ -n "$dm_dev" ]] || dm_dev=${!dev}
+ fi
- if [[ $(facet_fstype $facet) == zfs ]]; then
+ is_blkdev $facet $dm_dev || opts=$(csa_add "$opts" -o loop)
+
+ devicelabel=$(do_facet ${facet} "$E2LABEL $dm_dev");;
+ zfs)
# import ZFS storage pool
import_zpool $facet || return ${PIPESTATUS[0]}
- fi
- case $fstype in
- ldiskfs)
- devicelabel=$(do_facet ${facet} "$E2LABEL ${!dev}");;
- zfs)
devicelabel=$(do_facet ${facet} "$ZFS get -H -o value \
- lustre:svname ${!dev}");;
+ lustre:svname $dm_dev");;
*)
error "unknown fstype!";;
esac
- echo "Starting ${facet}: $opts ${!dev} $mntpt"
+ echo "Starting ${facet}: $opts $dm_dev $mntpt"
# for testing LU-482 error handling in mount_facets() and test_0a()
if [ -f $TMP/test-lu482-trigger ]; then
RC=2
else
- do_facet ${facet} "mkdir -p $mntpt; $MOUNT_CMD $opts \
- ${!dev} $mntpt"
+ do_facet ${facet} \
+ "mkdir -p $mntpt; $MOUNT_CMD $opts $dm_dev $mntpt"
RC=${PIPESTATUS[0]}
fi
if [ $RC -ne 0 ]; then
- echo "Start of ${!dev} on ${facet} failed ${RC}"
+ echo "Start of $dm_dev on ${facet} failed ${RC}"
return $RC
fi
fi
if [[ $opts =~ .*nosvc.* ]]; then
- echo "Start ${!dev} without service"
+ echo "Start $dm_dev without service"
else
case $fstype in
ldiskfs)
- wait_update_facet ${facet} "$E2LABEL ${!dev} \
+ wait_update_facet ${facet} "$E2LABEL $dm_dev \
2>/dev/null | grep -E ':[a-zA-Z]{3}[0-9]{4}'" \
- "" || error "${!dev} failed to initialize!";;
+ "" || error "$dm_dev failed to initialize!";;
zfs)
wait_update_facet ${facet} "$ZFS get -H -o value \
- lustre:svname ${!dev} 2>/dev/null | \
+ lustre:svname $dm_dev 2>/dev/null | \
grep -E ':[a-zA-Z]{3}[0-9]{4}'" "" ||
- error "${!dev} failed to initialize!";;
+ error "$dm_dev failed to initialize!";;
*)
error "unknown fstype!";;
fi
- label=$(devicelabel ${facet} ${!dev})
- [ -z "$label" ] && echo no label for ${!dev} && exit 1
+ label=$(devicelabel ${facet} $dm_dev)
+ [ -z "$label" ] && echo no label for $dm_dev && exit 1
eval export ${facet}_svc=${label}
echo Started ${label}
+ export_dm_dev $facet $dm_dev
+
return $RC
}
shift
local device=$1
shift
- eval export ${facet}_dev=${device}
+ local dev_alias=$(facet_device_alias $facet)
+
+ eval export ${dev_alias}_dev=${device}
eval export ${facet}_opt=\"$@\"
- local varname=${facet}failover_dev
+ local varname=${dev_alias}failover_dev
if [ -n "${!varname}" ] ; then
- eval export ${facet}failover_dev=${!varname}
+ eval export ${dev_alias}failover_dev=${!varname}
else
- eval export ${facet}failover_dev=$device
+ eval export ${dev_alias}failover_dev=$device
fi
local mntpt=$(facet_mntpt $facet)
}
stop() {
- local running
- local facet=$1
- shift
- local HOST=`facet_active_host $facet`
- [ -z $HOST ] && echo stop: no host for $facet && return 0
-
- local mntpt=$(facet_mntpt $facet)
- running=$(do_facet ${facet} "grep -c $mntpt' ' /proc/mounts") || true
- if [ ${running} -ne 0 ]; then
- echo "Stopping $mntpt (opts:$@) on $HOST"
- do_facet ${facet} $UMOUNT $@ $mntpt
- fi
+ local running
+ local facet=$1
+ shift
+ local HOST=$(facet_active_host $facet)
+ [[ -z $HOST ]] && echo stop: no host for $facet && return 0
+
+ local mntpt=$(facet_mntpt $facet)
+ running=$(do_facet ${facet} "grep -c $mntpt' ' /proc/mounts || true")
+ if [ ${running} -ne 0 ]; then
+ echo "Stopping $mntpt (opts:$@) on $HOST"
+ do_facet ${facet} $UMOUNT $@ $mntpt
+ fi
# umount should block, but we should wait for unrelated obd's
# like the MGS or MGC to also stop.
if [[ $(facet_fstype $facet) == zfs ]]; then
# export ZFS storage pool
[ "$KEEP_ZPOOL" = "true" ] || export_zpool $facet
+ elif dm_flakey_supported $facet; then
+ local host=${facet}_HOST
+ local failover_host=${facet}failover_HOST
+ if [[ -n ${!failover_host} && ${!failover_host} != ${!host} ]]||
+ $CLEANUP_DM_DEV || [[ $facet = fs* ]]; then
+ dm_cleanup_dev $facet
+ fi
fi
}
fi
}
+# Mount the file system on the MGS
+mount_mgs_client() {
+ do_facet mgs "mkdir -p $MOUNT"
+ zconf_mount $mgs_HOST $MOUNT $MOUNT_OPTS ||
+ error "unable to mount $MOUNT on MGS"
+}
+
+# Unmount the file system on the MGS
+umount_mgs_client() {
+ zconf_umount $mgs_HOST $MOUNT
+ do_facet mgs "rm -rf $MOUNT"
+}
+
# nodes is comma list
sanity_mount_check_nodes () {
local nodes=$1
}
shutdown_facet() {
- local facet=$1
+ local facet=$1
+ local affected_facet
+ local affected_facets
+
+ if [[ "$FAILURE_MODE" = HARD ]]; then
+ if [[ $(facet_fstype $facet) = ldiskfs ]] &&
+ dm_flakey_supported $facet; then
+ affected_facets=$(affected_facets $facet)
+ for affected_facet in ${affected_facets//,/ }; do
+ unexport_dm_dev $affected_facet
+ done
+ fi
- if [ "$FAILURE_MODE" = HARD ]; then
- shutdown_node_hard $(facet_active_host $facet)
- else
- stop $facet
- fi
+ shutdown_node_hard $(facet_active_host $facet)
+ else
+ stop $facet
+ fi
}
reboot_node() {
sync_all_data() {
do_nodes $(comma_list $(mdts_nodes)) \
- "lctl set_param -n osd*.*MDT*.force_sync=1"
+ "lctl set_param -n os[cd]*.*MDT*.force_sync=1"
do_nodes $(comma_list $(osts_nodes)) \
"lctl set_param -n osd*.*OS*.force_sync=1" 2>&1 |
grep -v 'Found no match'
}
wait_zfs_commit() {
+ local zfs_wait=${2:-5}
+
# the occupied disk space will be released
- # only after DMUs are committed
+ # only after TXGs are committed
if [[ $(facet_fstype $1) == zfs ]]; then
- echo "sleep $2 for ZFS OSD"
- sleep $2
+ echo "sleep $zfs_wait for ZFS $(facet_fstype $1)"
+ sleep $zfs_wait
fi
}
wait_delete_completed_mds() {
- local MAX_WAIT=${1:-20}
- # for ZFS, waiting more time for DMUs to be committed
- local ZFS_WAIT=${2:-5}
+ local max_wait=${1:-20}
local mds2sync=""
local stime=$(date +%s)
local etime
mds2sync="$mds2sync $node"
done
if [ -z "$mds2sync" ]; then
- wait_zfs_commit $SINGLEMDS $ZFS_WAIT
- return
+ wait_zfs_commit $SINGLEMDS
+ return 0
fi
mds2sync=$(comma_list $mds2sync)
# sync MDS transactions
- do_nodes $mds2sync "$LCTL set_param -n osd*.*MD*.force_sync 1"
+ do_nodes $mds2sync "$LCTL set_param -n os[cd]*.*MD*.force_sync 1"
# wait till all changes are sent and commmitted by OSTs
# for ldiskfs space is released upon execution, but DMU
# do this upon commit
local WAIT=0
- while [[ $WAIT -ne $MAX_WAIT ]]; do
+ while [[ $WAIT -ne $max_wait ]]; do
changes=$(do_nodes $mds2sync \
"$LCTL get_param -n osc.*MDT*.sync_*" | calc_sum)
#echo "$node: $changes changes on all"
if [[ $changes -eq 0 ]]; then
- wait_zfs_commit $SINGLEMDS $ZFS_WAIT
- return
+ wait_zfs_commit $SINGLEMDS
+
+ # the occupied disk space will be released
+ # only after TXGs are committed
+ wait_zfs_commit ost1
+ return 0
fi
sleep 1
- WAIT=$(( WAIT + 1))
+ WAIT=$((WAIT + 1))
done
etime=$(date +%s)
echo "Delete is not completed in $((etime - stime)) seconds"
do_nodes $mds2sync "$LCTL get_param osc.*MDT*.sync_*"
+ return 1
}
wait_for_host() {
done
# show which nodes are not finished.
+ cmd=$(echo $cmd | sed 's/-n//')
do_nodes $list "$cmd"
echo "$facet recovery node $i not done in $WAIT_TIMEOUT sec. $STATUS"
return 1
wait_delete_completed() {
wait_delete_completed_mds $1 || return $?
- wait_destroy_complete
+ wait_destroy_complete || return $?
}
wait_exit_ST () {
local running
# conf-sanity 31 takes a long time cleanup
while [ $WAIT -lt 300 ]; do
- running=$(do_facet ${facet} "lsmod | grep lnet > /dev/null && lctl dl | grep ' ST '") || true
+ running=$(do_facet ${facet} "lsmod | grep lnet > /dev/null &&
+lctl dl | grep ' ST ' || true")
[ -z "${running}" ] && return 0
echo "waited $WAIT for${running}"
[ $INTERVAL -lt 64 ] && INTERVAL=$((INTERVAL + INTERVAL))
# handled by stop() and mount_facet() separately, which are used
# inside fail() and fail_abort().
#
- do_facet $facet $LCTL --device ${!svc} readonly
+ set_dev_readonly $facet
do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
$LCTL mark "local REPLAY BARRIER on ${!svc}"
}
local svc=${facet}_svc
echo Replay barrier on ${!svc}
do_facet $facet $LCTL --device ${!svc} notransno
- do_facet $facet $LCTL --device ${!svc} readonly
+ set_dev_readonly $facet
do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
$LCTL mark "local REPLAY BARRIER on ${!svc}"
}
local svc=${facet}_svc
echo Replay barrier on ${!svc}
do_facet $facet $LCTL --device ${!svc} notransno
- do_facet $facet $LCTL --device ${!svc} readonly
+ set_dev_readonly $facet
do_facet $facet $LCTL mark "$facet REPLAY BARRIER on ${!svc}"
$LCTL mark "local REPLAY BARRIER on ${!svc}"
}
local clients=${CLIENTS:-$HOSTNAME}
facet_failover $* || error "failover: $?"
- wait_clients_import_state "$clients" "$facets" FULL
+ # to initiate all OSC idling connections
+ clients_up
+ wait_clients_import_state "$clients" "$facets" "\(FULL\|IDLE\)"
clients_up || error "post-failover stat: $?"
}
$myPDSH $HOST "$LCTL mark \"$@\"" > /dev/null 2>&1 || :
fi
- if [ "$myPDSH" = "rsh" ]; then
-# we need this because rsh does not return exit code of an executed command
- local command_status="$TMP/cs"
- rsh $HOST ":> $command_status"
- rsh $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin;
- cd $RPWD; LUSTRE=\"$RLUSTRE\" sh -c \"$@\") ||
- echo command failed >$command_status"
- [ -n "$($myPDSH $HOST cat $command_status)" ] && return 1 || true
- return 0
- fi
+ if [[ "$myPDSH" == "rsh" ]] ||
+ [[ "$myPDSH" == *pdsh* && "$myPDSH" != *-S* ]]; then
+ # we need this because rsh and pdsh do not return
+ # exit code of an executed command
+ local command_status="$TMP/cs"
+ eval $myPDSH $HOST ":> $command_status"
+ eval $myPDSH $HOST "(PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests;
+ PATH=\$PATH:/sbin:/usr/sbin;
+ cd $RPWD;
+ LUSTRE=\"$RLUSTRE\" sh -c \"$@\") ||
+ echo command failed >$command_status"
+ [[ -n "$($myPDSH $HOST cat $command_status)" ]] && return 1 ||
+ return 0
+ fi
if $verbose ; then
# print HOSTNAME for myPDSH="no_dsh"
case $fstype in
ldiskfs )
+ local dev=ost${num}_dev
+ [[ -n ${!dev} ]] && eval DEVPTR=${!dev} ||
#if $OSTDEVn isn't defined, default is $OSTDEVBASE + num
eval DEVPTR=${!DEVNAME:=${OSTDEVBASE}${num}};;
zfs )
case $fstype in
ldiskfs )
+ local dev=mds${num}_dev
+ [[ -n ${!dev} ]] && eval DEVPTR=${!dev} ||
#if $MDSDEVn isn't defined, default is $MDSDEVBASE{n}
eval DEVPTR=${!DEVNAME:=${MDSDEVBASE}${num}};;
zfs )
case $fstype in
ldiskfs )
if [ $(facet_host mgs) = $(facet_host mds1) ] &&
- ( [ -z "$MGSDEV" ] || [ $MGSDEV = $(mdsdevname 1) ] ); then
+ ( [ -z "$MGSDEV" ] || [ $MGSDEV = $MDSDEV1 ] ); then
DEVPTR=$(mdsdevname 1)
else
+ [[ -n $mgs_dev ]] && DEVPTR=$mgs_dev ||
DEVPTR=$MGSDEV
fi;;
zfs )
local dev=$(facet_device $facet)
local mnt=${2:-$(facet_mntpt $facet)}
local opts
+ local dm_dev=$dev
- if ! do_facet $facet test -b $dev; then
- opts="-o loop"
+ if dm_flakey_supported $facet; then
+ dm_dev=$(dm_create_dev $facet $dev)
+ [[ -n "$dm_dev" ]] || dm_dev=$dev
fi
- do_facet $facet mount -t ldiskfs $opts $dev $mnt
+ is_blkdev $facet $dm_dev || opts=$(csa_add "$opts" -o loop)
+ export_dm_dev $facet $dm_dev
+
+ do_facet $facet mount -t ldiskfs $opts $dm_dev $mnt
}
unmount_ldiskfs() {
nfs_client_mode && return
cifs_client_mode && return
- stopall $*
cleanup_echo_devs
+ CLEANUP_DM_DEV=true stopall $*
unload_modules
cleanup_sk
if [ $type == MDS ]; then
opts+=${MDSCAPA:+" --param-mdt.capa=$MDSCAPA"}
- opts+=${STRIPE_BYTES:+" --param=lov.stripesize=$STRIPE_BYTES"}
- opts+=${STRIPES_PER_OBJ:+" --param=lov.stripecount=$STRIPES_PER_OBJ"}
+ opts+=${DEF_STRIPE_SIZE:+" --param=lov.stripesize=$DEF_STRIPE_SIZE"}
+ opts+=${DEF_STRIPE_COUNT:+" --param=lov.stripecount=$DEF_STRIPE_COUNT"}
opts+=${L_GETIDENTITY:+" --param=mdt.identity_upcall=$L_GETIDENTITY"}
if [ $fstype == ldiskfs ]; then
}
mountmds() {
+ local num
+ local devname
+ local host
+ local varname
for num in $(seq $MDSCOUNT); do
- DEVNAME=$(mdsdevname $num)
- start mds$num $DEVNAME $MDS_MOUNT_OPTS
-
- # We started mds, now we should set failover variables properly.
- # Set mds${num}failover_HOST if unset (the default
- # failnode).
- local varname=mds${num}failover_HOST
- if [ -z "${!varname}" ]; then
- eval mds${num}failover_HOST=$(facet_host mds$num)
- fi
-
+ devname=$(mdsdevname $num)
+ start mds$num $devname $MDS_MOUNT_OPTS
+
+ # We started mds$num, now we should set mds${num}_HOST
+ # and mds${num}failover_HOST variables properly if they
+ # are not set.
+ host=$(facet_host mds$num)
+ for varname in mds${num}_HOST mds${num}failover_HOST; do
+ if [[ -z "${!varname}" ]]; then
+ eval $varname=$host
+ fi
+ done
if [ $IDENTITY_UPCALL != "default" ]; then
switch_identity $num $IDENTITY_UPCALL
fi
}
mountoss() {
+ local num
+ local devname
+ local host
+ local varname
for num in $(seq $OSTCOUNT); do
- DEVNAME=$(ostdevname $num)
- start ost$num $DEVNAME $OST_MOUNT_OPTS
-
- # We started ost$num, now we should set ost${num}failover
- # variable properly. Set ost${num}failover_HOST if it is not
- # set (the default failnode).
- varname=ost${num}failover_HOST
- if [ -z "${!varname}" ]; then
- eval ost${num}failover_HOST=$(facet_host ost${num})
- fi
-
+ devname=$(ostdevname $num)
+ start ost$num $devname $OST_MOUNT_OPTS
+
+ # We started ost$num, now we should set ost${num}_HOST
+ # and ost${num}failover_HOST variables properly if they
+ # are not set.
+ host=$(facet_host ost$num)
+ for varname in ost${num}_HOST ost${num}failover_HOST; do
+ if [[ -z "${!varname}" ]]; then
+ eval $varname=$host
+ fi
+ done
done
}
[ "$arg1" = "server_only" ] && return
fi
mount_client $MOUNT
- [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT
+ if [ -n "$CLIENTS" ]; then
+ zconf_mount_clients $CLIENTS $MOUNT
+ fi
clients_up
if [ "$MOUNT_2" ]; then
mount_client $MOUNT2
- [ -n "$CLIENTS" ] && zconf_mount_clients $CLIENTS $MOUNT2
+ if [ -n "$CLIENTS" ]; then
+ zconf_mount_clients $CLIENTS $MOUNT2
+ fi
fi
}
# 1.
# both MOUNT and MOUNT2 are not mounted
if ! is_mounted $MOUNT && ! is_mounted $MOUNT2; then
- [ "$REFORMAT" = "yes" ] && formatall
+ [ "$REFORMAT" = "yes" ] && CLEANUP_DM_DEV=true formatall
# setupall mounts both MOUNT and MOUNT2 (if MOUNT_2 is set)
setupall
is_mounted $MOUNT || error "NAME=$NAME not mounted"
if [ -n "$(grep "DNE mode isn't supported" $log)" ]; then
rm -f $log
if [ $MDSCOUNT -gt 1 ]; then
- skip "DNE mode isn't supported!"
+ skip_noexit "DNE mode isn't supported!"
cleanupall
exit_status
else
cleanup_mount $MOUNT2
fi
- if [ "$I_MOUNTED" = "yes" ]; then
+ if [[ "$I_MOUNTED" = "yes" ]] && ! $AUSTER_CLEANUP; then
cleanupall -f || error "cleanup failed"
unset I_MOUNTED
fi
}
check_network() {
- local host=$1
- local max=$2
- local sleep=${3:-5}
+ local host=$1
+ local max=$2
+ local sleep=${3:-5}
- echo `date +"%H:%M:%S (%s)"` waiting for $host network $max secs ...
- if ! wait_for_function --quiet "ping -c 1 -w 3 $host" $max $sleep ; then
- echo "Network not available!"
- exit 1
- fi
+ [ "$host" = "$HOSTNAME" ] && return 0
+
+ echo "$(date +'%H:%M:%S (%s)') waiting for $host network $max secs ..."
+ if ! wait_for_function --quiet "ping -c 1 -w 3 $host" $max $sleep ; then
+ echo "Network not available!"
+ exit 1
+ fi
- echo `date +"%H:%M:%S (%s)"` network interface is UP
+ echo "$(date +'%H:%M:%S (%s)') network interface is UP"
}
no_dsh() {
# Test interface
##################################
+# usage: stack_trap arg sigspec
+#
+# stack_trap() behaves like bash's built-in trap, except that it "stacks" the
+# command ``arg`` on top of previously defined commands for ``sigspec`` instead
+# of overwriting them.
+# stacked traps are executed in reverse order of their registration
+#
+# arg and sigspec have the same meaning as in man (1) trap
+stack_trap()
+{
+ local arg="$1"
+ local sigspec="$2"
+
+ # Use "trap -p" to get the quoting right
+ local old_trap="$(trap -p "$sigspec")"
+ # Append ";" and remove the leading "trap -- '" added by "trap -p"
+ old_trap="${old_trap:+"; ${old_trap#trap -- \'}"}"
+
+ # Once again, use "trap -p" to get the quoting right
+ local new_trap="$(trap -- "$arg" "$sigspec"
+ trap -p "$sigspec"
+ trap -- '' "$sigspec")"
+
+ # Remove the trailing "' $sigspec" part added by "trap -p" and merge
+ #
+ # The resulting string should be safe to "eval" as it is (supposedly
+ # correctly) quoted by "trap -p"
+ eval "${new_trap%\' $sigspec}${old_trap:-"' $sigspec"}"
+}
+
error_noexit() {
report_error "$@"
}
$FAIL_ON_SKIP_ENV && error false $@ || skip $@
}
-skip() {
+skip_noexit() {
echo
log " SKIP: $TESTSUITE $TESTNAME $@"
echo "$TESTSUITE: SKIP: $TESTNAME $@" >> $TESTSUITELOG || true
}
+skip() {
+ skip_noexit $@
+ exit 0
+}
+
build_test_filter() {
- EXCEPT="$EXCEPT $(testslist_filter)"
+ EXCEPT="$EXCEPT $(testslist_filter)"
for O in $ONLY; do
if [[ $O = [0-9]*-[0-9]* ]]; then
ALWAYS_SKIPPED="y"
testname=EXCEPT_$1
if [ ${!testname}x != x ]; then
- TESTNAME=test_$1 skip "skipping excluded test $1"
+ TESTNAME=test_$1 skip_noexit "skipping excluded test $1"
return 0
fi
testname=EXCEPT_$base
if [ ${!testname}x != x ]; then
- TESTNAME=test_$1 skip "skipping excluded test $1 (base $base)"
+ TESTNAME=test_$1 skip_noexit "skipping excluded test $1 (base $base)"
return 0
fi
testname=EXCEPT_ALWAYS_$1
if [ ${!testname}x != x ]; then
- TESTNAME=test_$1 skip "skipping ALWAYS excluded test $1"
+ TESTNAME=test_$1 skip_noexit "skipping ALWAYS excluded test $1"
return 0
fi
testname=EXCEPT_ALWAYS_$base
if [ ${!testname}x != x ]; then
- TESTNAME=test_$1 skip "skipping ALWAYS excluded test $1 (base $base)"
+ TESTNAME=test_$1 skip_noexit "skipping ALWAYS excluded test $1 (base $base)"
return 0
fi
testname=EXCEPT_SLOW_$1
if [ ${!testname}x != x ]; then
- TESTNAME=test_$1 skip "skipping SLOW test $1"
+ TESTNAME=test_$1 skip_noexit "skipping SLOW test $1"
return 0
fi
testname=EXCEPT_SLOW_$base
if [ ${!testname}x != x ]; then
- TESTNAME=test_$1 skip "skipping SLOW test $1 (base $base)"
+ TESTNAME=test_$1 skip_noexit "skipping SLOW test $1 (base $base)"
return 0
fi
# sync all the data and make sure no pending data on server
do_nodes $clients sync
+ clients_up # initiate all idling connections
# get client grant
client_grant=$(do_nodes $clients \
server_grant=$(do_nodes $(comma_list $(osts_nodes)) \
"$LCTL get_param "\
"obdfilter.${FSNAME}-OST*.{tot_granted,tot_pending,grant_precreate}" |
- sed 's/=/ /'| awk '/tot_granted/{ total += $2 };
+ sed 's/=/ /'| awk '/tot_granted/{ total += $2 };
/tot_pending/{ total -= $2 };
/grant_precreate/{ total -= $2 };
END { printf("%0.0f", total) }')
}
calc_osc_kbytes () {
- df $MOUNT > /dev/null
+ $LFS df $MOUNT > /dev/null
$LCTL get_param -n osc.*[oO][sS][cC][-_][0-9a-f]*.$1 | calc_sum
}
local val
while IFS=" =" read facet name val; do
- do_facet $facet "$LCTL set_param -n $name $val"
+ do_facet $facet "$LCTL set_param -n $name=$val"
done
}
}
get_clientosc_proc_path() {
- echo "${1}-osc-*"
+ echo "${1}-osc-ffff*"
}
# If the 2.0 MDS was mounted on 1.8 device, then the OSC and LOV names
local mdt_label=$(convert_facet2label $mds_facet)
local mdt_index=$(echo $mdt_label | sed -e 's/^.*-//')
- if [ $(lustre_version_code $mds_facet) -le $(version_code 1.8.0) ] ||
- mds_on_old_device $mds_facet; then
- echo "${ost_label}-osc"
- elif [[ $ost_label = *OST* ]]; then
+ if [[ $ost_label = *OST* ]]; then
echo "${ost_label}-osc-${mdt_index}"
else
echo "${ost_label}-osp-${mdt_index}"
local i=0
CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2 | uniq)
- while [ "${CONN_STATE}" != "${expected}" ]; do
+ while ! echo "${CONN_STATE}" | egrep -q "^${expected}\$" ; do
if [ "${expected}" == "DISCONN" ]; then
# for disconn we can check after proc entry is removed
[ "x${CONN_STATE}" == "x" ] && return 0
fi
}
+wait_osc_import_ready() {
+ wait_osc_import_state $1 $2 "\(FULL\|IDLE\)"
+}
+
_wait_mgc_import_state() {
local facet=$1
local expected=$2
if [ $MDSCOUNT -gt 1 ]; then
for num in $(seq $MDSCOUNT); do
- wait_osc_import_state mds mds$num FULL
+ wait_osc_import_ready mds mds$num
done
fi
}
local LIBPATH="/usr/lib/lustre/tests:/usr/lib64/lustre/tests:"
local TESTPATH="$RLUSTRE/tests:"
local RPATH="PATH=${TESTPATH}${LIBPATH}${PATH}:/sbin:/bin:/usr/sbin:"
- do_nodesv $list "${RPATH} NAME=${NAME} sh rpc.sh $@ "
+ do_nodesv $list "${RPATH} NAME=${NAME} bash rpc.sh $@ "
}
wait_clients_import_state () {
local params=$(expand_list $params $proc_path)
done
- if ! do_rpc_nodes "$list" wait_import_state_mount $expected $params;
+ if ! do_rpc_nodes "$list" wait_import_state_mount "$expected" $params;
then
error "import is not in ${expected} state"
return 1
local fsname=${1%%.*}
local poolname=${1##$fsname.}
- trap "destroy_test_pools $fsname" EXIT
+ stack_trap "destroy_test_pools $fsname" EXIT
do_facet mgs lctl pool_new $1
local RC=$?
# get param should return err unless pool is created
# returns the canonical name for an ldiskfs device
ldiskfs_canon() {
- local dev="$1"
- local facet="$2"
-
- do_facet $facet "dv=\\\$(lctl get_param -n $dev);
-if foo=\\\$(lvdisplay -c \\\$dv 2>/dev/null); then
- echo dm-\\\${foo##*:};
-else
- echo \\\$(basename \\\$dv);
-fi;"
+ local dev="$1"
+ local facet="$2"
+
+ do_facet $facet "dv=\\\$($LCTL get_param -n $dev);
+ if foo=\\\$(lvdisplay -c \\\$dv 2>/dev/null); then
+ echo dm-\\\${foo##*:};
+ else
+ name=\\\$(basename \\\$dv);
+ if [[ \\\$name = *flakey* ]]; then
+ name=\\\$(lsblk -o NAME,KNAME |
+ awk /\\\$name/'{print \\\$NF}');
+ fi;
+ echo \\\$name;
+ fi;"
}
is_sanity_benchmark() {
if ! combined_mgs_mds ; then
[ $(facet_fstype mgs) != zfs ] &&
- skip "Lustre snapshot 1 only works for ZFS backend" &&
- exit 0
+ skip "Lustre snapshot 1 only works for ZFS backend"
local host=$(facet_active_host mgs)
local dir=$(dirname $(facet_vdevice mgs))
for num in `seq $MDSCOUNT`; do
[ $(facet_fstype mds$num) != zfs ] &&
- skip "Lustre snapshot 1 only works for ZFS backend" &&
- exit 0
+ skip "Lustre snapshot 1 only works for ZFS backend"
lss_gen_conf_one mds$num MDT $((num - 1)) ||
lss_err "generate lss conf (mds$num)"
for num in `seq $OSTCOUNT`; do
[ $(facet_fstype ost$num) != zfs ] &&
- skip "Lustre snapshot 1 only works for ZFS backend" &&
- exit 0
+ skip "Lustre snapshot 1 only works for ZFS backend"
lss_gen_conf_one ost$num OST $((num - 1)) ||
lss_err "generate lss conf (ost$num)"
do_facet mgs "cat $LSNAPSHOT_CONF"
}
+# Parse 'lfs getstripe -d <path_with_dir_name>' for non-composite dir
+parse_plain_dir_param()
+{
+ local invalues=($1)
+ local param=""
+
+ if [[ ${invalues[0]} =~ "stripe_count:" ]]; then
+ param="-c ${invalues[1]}"
+ fi
+ if [[ ${invalues[2]} =~ "stripe_size:" ]]; then
+ param="$param -S ${invalues[3]}"
+ fi
+ if [[ ${invalues[4]} =~ "pattern:" ]]; then
+ if [[ ${invalues[5]} =~ "stripe_offset:" ]]; then
+ param="$param -i ${invalues[6]}"
+ else
+ param="$param -L ${invalues[5]} -i ${invalues[7]}"
+ fi
+ elif [[ ${invalues[4]} =~ "stripe_offset:" ]]; then
+ param="$param -i ${invalues[5]}"
+ fi
+ echo "$param"
+}
+
parse_plain_param()
{
local line=$1
echo "-S $val"
elif [[ $line =~ ^"lmm_stripe_offset:" ]]; then
echo "-i $val"
+ elif [[ $line =~ ^"lmm_pattern:" ]]; then
+ echo "-L $val"
fi
}
local param=""
while read line; do
- if [[ -z $mode ]]; then
- if [[ $line =~ ^"stripe_count:" ]]; then
- mode="plain_dir"
- elif [[ $line =~ ^"lmm_stripe_count:" ]]; then
- mode="plain_file"
- elif [[ $line =~ ^"lcm_layout_gen:" ]]; then
- mode="pfl"
+ if [[ ! -z $line ]]; then
+ if [[ -z $mode ]]; then
+ if [[ $line =~ ^"stripe_count:" ]]; then
+ mode="plain_dir"
+ elif [[ $line =~ ^"lmm_stripe_count:" ]]; then
+ mode="plain_file"
+ elif [[ $line =~ ^"lcm_layout_gen:" ]]; then
+ mode="pfl"
+ fi
fi
- fi
- if [[ $mode = "plain_dir" ]]; then
- param=$(echo $line |
- awk '{printf("-c %d -S %d -i %d",$2,$4,$6)}')
- elif [[ $mode = "plain_file" ]]; then
- val=$(parse_plain_param "$line")
- [[ ! -z $val ]] && param="$param $val"
- elif [[ $mode = "pfl" ]]; then
- val=$(echo $line | awk '{print $2}')
- if [[ $line =~ ^"lcme_extent.e_end:" ]]; then
- if [[ $val = "EOF" ]]; then
- param="$param -E -1"
- else
- param="$param -E $val"
- fi
- elif [[ $line =~ ^"stripe_count:" ]]; then
- # pfl dir
- val=$(echo $line |
- awk '{printf("-c %d -S %d -i %d",$2,$4,$6)}')
- param="$param $val"
- else
- #pfl file
+ if [[ $mode = "plain_dir" ]]; then
+ param=$(parse_plain_dir_param "$line")
+ elif [[ $mode = "plain_file" ]]; then
val=$(parse_plain_param "$line")
[[ ! -z $val ]] && param="$param $val"
+ elif [[ $mode = "pfl" ]]; then
+ val=$(echo $line | awk '{print $2}')
+ if [[ $line =~ ^"lcme_extent.e_end:" ]]; then
+ if [[ $val = "EOF" ]]; then
+ param="$param -E -1"
+ else
+ param="$param -E $val"
+ fi
+ elif [[ $line =~ ^"stripe_count:" ]]; then
+ # pfl dir
+ val=$(parse_plain_dir_param "$line")
+ param="$param $val"
+ else
+ #pfl file
+ val=$(parse_plain_param "$line")
+ [[ ! -z $val ]] && param="$param $val"
+ fi
fi
fi
done
local param=$($LFS getstripe -d $1 | parse_layout_param)
echo "$param"
}
+
+lfsck_verify_pfid()
+{
+ local f
+ local rc=0
+
+ # Cancel locks before setting lfsck_verify_pfid so that errors are more
+ # controllable
+ cancel_lru_locks mdc
+ cancel_lru_locks osc
+
+ # make sure PFID is set correctly for files
+ do_nodes $(comma_list $(osts_nodes)) \
+ "$LCTL set_param -n obdfilter.${FSNAME}-OST*.lfsck_verify_pfid=1"
+
+ for f in "$@"; do
+ cat $f &> /dev/nullA ||
+ { rc=$?; echo "verify $f failed"; break; }
+ done
+
+ do_nodes $(comma_list $(osts_nodes)) \
+ "$LCTL set_param -n obdfilter.${FSNAME}-OST*.lfsck_verify_pfid=0"
+ return $rc
+}
+
+# check that clients "oscs" was evicted after "before"
+check_clients_evicted() {
+ local before=$1
+ shift
+ local oscs=${@}
+ local osc
+ local rc=0
+
+ for osc in $oscs; do
+ ((rc++))
+ echo "Check state for $osc"
+ local evicted=$(do_facet client $LCTL get_param osc.$osc.state |
+ tail -n 3 | awk -F"[ [,]" \
+ '/EVICTED ]$/ { if (mx<$5) {mx=$5;} } END { print mx }')
+ if (($? == 0)) && (($evicted > $before)); then
+ echo "$osc is evicted at $evicted"
+ ((rc--))
+ fi
+ done
+
+ [ $rc -eq 0 ] || error "client not evicted from OST"
+}
+
+# check that clients OSCS current_state is FULL
+check_clients_full() {
+ local timeout=$1
+ shift
+ local oscs=${@}
+
+ for osc in $oscs; do
+ wait_update_facet client \
+ "lctl get_param -n osc.$osc.state |
+ grep 'current_state: FULL'" \
+ "current_state: FULL" $timeout
+ [ $? -eq 0 ] || error "$osc state is not FULL"
+ done
+}
+
+#Changelogs
+__changelog_deregister() {
+ local facet=$1
+ local mdt="$(facet_svc $facet)"
+ local cl_user=$2
+ local rc=0
+
+ # skip cleanup if no user registered for this MDT
+ [ -z "$cl_user" ] && echo "$mdt: no changelog user" && return 0
+ # user is no longer registered, skip cleanup
+ changelog_users "$facet" | grep -q "$cl_user" ||
+ { echo "$mdt: changelog user '$cl_user' not found"; return 0; }
+
+ # From this point, if any operation fails, it is an error
+ __changelog_clear $facet $cl_user 0 ||
+ error_noexit "$mdt: changelog_clear $cl_user 0 fail: $rc"
+ do_facet $facet $LCTL --device $mdt changelog_deregister $cl_user ||
+ error_noexit "$mdt: changelog_deregister '$cl_user' fail: $rc"
+}
+
+declare -Ax CL_USERS
+changelog_register() {
+ for M in $(seq $MDSCOUNT); do
+ local facet=mds$M
+ local mdt="$(facet_svc $facet)"
+ local cl_mask
+
+ cl_mask=$(do_facet $facet $LCTL get_param \
+ mdd.${mdt}.changelog_mask -n)
+ stack_trap "do_facet $facet $LCTL \
+ set_param mdd.$mdt.changelog_mask=\'$cl_mask\' -n" EXIT
+ do_facet $facet $LCTL set_param mdd.$mdt.changelog_mask=+hsm ||
+ error "$mdt: changelog_mask=+hsm failed: $?"
+
+ local cl_user
+ cl_user=$(do_facet $facet \
+ $LCTL --device $mdt changelog_register -n) ||
+ error "$mdt: register changelog user failed: $?"
+ stack_trap "__changelog_deregister $facet $cl_user" EXIT
+
+ stack_trap "CL_USERS[$facet]='${CL_USERS[$facet]}'" EXIT
+ # Bash does not support nested arrays, but the format of a
+ # cl_user is constrained enough to use whitespaces as separators
+ CL_USERS[$facet]+="$cl_user "
+ done
+ echo "Registered $MDSCOUNT changelog users: '${CL_USERS[@]% }'"
+}
+
+changelog_deregister() {
+ local cl_user
+ # bash assoc arrays do not guarantee to list keys in created order
+ # so reorder to get same order than in changelog_register()
+ local cl_facets=$(echo "${!CL_USERS[@]}" | tr " " "\n" | sort |
+ tr "\n" " ")
+
+ for facet in $cl_facets; do
+ for cl_user in ${CL_USERS[$facet]}; do
+ __changelog_deregister $facet $cl_user || return $?
+ done
+ unset CL_USERS[$facet]
+ done
+}
+
+changelog_users() {
+ local facet=$1
+ local service=$(facet_svc $facet)
+
+ do_facet $facet $LCTL get_param -n mdd.$service.changelog_users
+}
+
+changelog_user_rec() {
+ local facet=$1
+ local cl_user=$2
+ local service=$(facet_svc $facet)
+
+ changelog_users $facet | awk '$1 == "'$cl_user'" { print $2 }'
+}
+
+changelog_chmask() {
+ local mask=$1
+
+ do_nodes $(comma_list $(mdts_nodes)) \
+ $LCTL set_param mdd.*.changelog_mask="$mask"
+}
+
+# usage: __changelog_clear FACET CL_USER [+]INDEX
+__changelog_clear()
+{
+ local facet=$1
+ local mdt="$(facet_svc $facet)"
+ local cl_user=$2
+ local -i rec
+
+ case "$3" in
+ +*)
+ # Remove the leading '+'
+ rec=${3:1}
+ rec+=$(changelog_user_rec $facet $cl_user)
+ ;;
+ *)
+ rec=$3
+ ;;
+ esac
+
+ if [ $rec -eq 0 ]; then
+ echo "$mdt: clear the changelog for $cl_user of all records"
+ else
+ echo "$mdt: clear the changelog for $cl_user to record #$rec"
+ fi
+ $LFS changelog_clear $mdt $cl_user $rec
+}
+
+# usage: changelog_clear [+]INDEX
+#
+# If INDEX is prefixed with '+', increment every changelog user's record index
+# by INDEX. Otherwise, clear the changelog up to INDEX for every changelog
+# users.
+changelog_clear() {
+ local rc
+ # bash assoc arrays do not guarantee to list keys in created order
+ # so reorder to get same order than in changelog_register()
+ local cl_facets=$(echo "${!CL_USERS[@]}" | tr " " "\n" | sort |
+ tr "\n" " ")
+
+ for facet in $cl_facets; do
+ for cl_user in ${CL_USERS[$facet]}; do
+ __changelog_clear $facet $cl_user $1 || rc=${rc:-$?}
+ done
+ done
+
+ return ${rc:-0}
+}
+
+changelog_dump() {
+ for M in $(seq $MDSCOUNT); do
+ local facet=mds$M
+ local mdt="$(facet_svc $facet)"
+
+ $LFS changelog $mdt | sed -e 's/^/'$mdt'./'
+ done
+}
+
+changelog_extract_field() {
+ local cltype=$1
+ local file=$2
+ local identifier=$3
+
+ changelog_dump | gawk "/$cltype.*$file$/ {
+ print gensub(/^.* "$identifier'(\[[^\]]*\]).*$/,"\\1",1)}' |
+ tail -1
+}
+
+# Prints a changelog record produced by "lfs changelog" as an associative array
+#
+# Example:
+# $> changelog2array 16 01CREAT 10:28:46.968438800 2018.03.09 0x0 \
+# t=[0x200000401:0x10:0x0] j=touch.501 ef=0xf u=501:501 \
+# nid=0@lo p=[0x200000007:0x1:0x0] blob
+# ([index]='16' [type]='CREAT' [time]='10:28:46.968438800'
+# [date]='2018.03.09' [flags]=0x0 ['target-fid']='[0x200000401:0x10:0x0]'
+# ['jobid']='touch.501' ['extra-flags']='0x0f' [uid]='0' ['gid']='0'
+# ['nid']='0@lo' ['parent-fid']='[0x200000007:0x1:0x0]')
+#
+# Note that the changelog record is not quoted
+# Also note that the line breaks in the output were only added for readability
+#
+# Typically, you want to eval the output of the command to fill an actual
+# associative array, like this:
+# $> eval declare -A changelog=$(changelog2array $entry)
+#
+# It can then be accessed like any bash associative array:
+# $> echo "${changelog[index]}" "${changelog[type]}" "${changelog[flags]}"
+# 16 CREAT 0x0
+# $> echo "${changelog[uid]}":"${changelog[gid]}"
+# 501:501
+#
+changelog2array()
+{
+ # Start the array
+ printf '('
+
+ # A changelog, as printed by "lfs changelog" typically looks like this:
+ # <index> <type> <time> <date> <flags> <key1=value1> <key2=value2> ...
+
+ # Parse the positional part of the changelog
+
+ # changelog_dump() prefixes records with their mdt's name
+ local index="${1##*.}"
+
+ printf "[index]='%s' [type]='%s' [time]='%s' [date]='%s' [flags]='%s'" \
+ "$index" "${2:2}" "$3" "$4" "$5"
+
+ # Parse the key/value part of the changelog
+ for arg in "${@:5}"; do
+ # Check it matches a key=value syntax
+ [[ "$arg" =~ ^[[:alpha:]]+= ]] || continue
+
+ local key="${arg%%=*}"
+ local value="${arg#*=}"
+
+ case "$key" in
+ u)
+ # u is actually for uid AND gid: u=UID:GID
+ printf " [uid]='%s'" "${value%:*}"
+ key=gid
+ value="${value#*:}"
+ ;;
+ t)
+ key=target-fid
+ value="${value#[}"
+ value="${value%]}"
+ ;;
+ j)
+ key=jobid
+ ;;
+ p)
+ key=parent-fid
+ value="${value#[}"
+ value="${value%]}"
+ ;;
+ ef)
+ key=extra-flags
+ ;;
+ m)
+ key=mode
+ ;;
+ x)
+ key=xattr
+ ;;
+ *)
+ ;;
+ esac
+
+ printf " ['%s']='%s'" "$key" "$value"
+ done
+
+ # end the array
+ printf ')'
+}
+
+# Format and print a changelog record
+#
+# Interpreted sequences are:
+# %% a single %
+# %f the "flags" attribute of a changelog record
+__changelog_printf()
+{
+ local format="$1"
+
+ local -i i
+ for ((i = 0; i < ${#format}; i++)); do
+ local char="${format:$i:1}"
+ if [ "$char" != % ]; then
+ printf '%c' "$char"
+ continue
+ fi
+
+ i+=1
+ char="${format:$i:1}"
+ case "$char" in
+ f)
+ printf '%s' "${changelog[flags]}"
+ ;;
+ %)
+ printf '%'
+ ;;
+ esac
+ done
+ printf '\n'
+}
+
+# Filter changelog records
+changelog_find()
+{
+ local -A filter
+ local action='print'
+ local format
+
+ while [ $# -gt 0 ]; do
+ case "$1" in
+ -print)
+ action='print'
+ ;;
+ -printf)
+ action='printf'
+ format="$2"
+ shift
+ ;;
+ -*)
+ filter[${1#-}]="$2"
+ shift
+ ;;
+ esac
+ shift
+ done
+
+ local found=false
+ local record
+ changelog_dump | { while read -r record; do
+ eval local -A changelog=$(changelog2array $record)
+ for key in "${!filter[@]}"; do
+ case "$key" in
+ *)
+ [ "${changelog[$key]}" == "${filter[$key]}" ]
+ ;;
+ esac || continue 2
+ done
+
+ found=true
+
+ case "${action:-print}" in
+ print)
+ printf '%s\n' "$record"
+ ;;
+ printf)
+ __changelog_printf "$format"
+ ;;
+ esac
+ done; $found; }
+}
+
+restore_layout() {
+ local dir=$1
+ local layout=$2
+
+ [ ! -d "$dir" ] && return
+
+ [ -z "$layout" ] && {
+ $LFS setstripe -d $dir || error "error deleting stripe '$dir'"
+ return
+ }
+
+ setfattr -n trusted.lov -v $layout $dir ||
+ error "error restoring layout '$layout' to '$dir'"
+}
+
+# save the layout of a directory, the returned string will be used by
+# restore_layout() to restore the layout
+save_layout() {
+ local dir=$1
+ local str=$(getfattr -n trusted.lov --absolute-names -e hex $dir \
+ 2> /dev/null | awk -F'=' '/trusted.lov/{ print $2 }')
+ echo "$str"
+}
+
+# save layout of a directory and restore it at exit
+save_layout_restore_at_exit() {
+ local dir=$1
+ local layout=$(save_layout $dir)
+
+ stack_trap "restore_layout $dir $layout" EXIT
+}
git://git.whamcloud.com / fs / lustre-release.git / blobdiff