}
init_test_env() {
- export LUSTRE=`absolute_path $LUSTRE`
- export TESTSUITE=`basename $0 .sh`
- export TEST_FAILED=false
- export FAIL_ON_SKIP_ENV=${FAIL_ON_SKIP_ENV:-false}
+ export LUSTRE=$(absolute_path $LUSTRE)
+ export TESTSUITE=$(basename $0 .sh)
+ export TEST_FAILED=false
+ export FAIL_ON_SKIP_ENV=${FAIL_ON_SKIP_ENV:-false}
+ export RPC_MODE=${RPC_MODE:-false}
export MKE2FS=$MKE2FS
if [ -z "$MKE2FS" ]; then
shift $((OPTIND - 1))
ONLY=${ONLY:-$*}
- # print the durations of each test if "true"
- DDETAILS=${DDETAILS:-false}
- [ "$TESTSUITELOG" ] && rm -f $TESTSUITELOG || true
- rm -f $TMP/*active
+ # print the durations of each test if "true"
+ DDETAILS=${DDETAILS:-false}
+ [ "$TESTSUITELOG" ] && rm -f $TESTSUITELOG || true
+ if ! $RPC_MODE; then
+ rm -f $TMP/*active
+ fi
}
check_cpt_number() {
return 0
}
+fs_log_size() {
+ local facet=${1:-$SINGLEMDS}
+ local fstype=$(facet_fstype $facet)
+ local size=0
+ case $fstype in
+ ldiskfs) size=50;; # largest seen is 44, leave some headroom
+ zfs) size=256;;
+ esac
+
+ echo -n $size
+}
+
check_gss_daemon_nodes() {
local list=$1
dname=$2
}
#
-# This and set_obdfilter_param() shall be used to access OSD parameters
+# This and set_osd_param() shall be used to access OSD parameters
# once existed under "obdfilter":
#
# mntdev
# read_cache_enable
# writethrough_cache_enable
#
-get_obdfilter_param() {
+get_osd_param() {
local nodes=$1
local device=${2:-$FSNAME-OST*}
local name=$3
osd-*.$device.$name 2>&1" | grep -v 'Found no match'
}
-set_obdfilter_param() {
+set_osd_param() {
local nodes=$1
local device=${2:-$FSNAME-OST*}
local name=$3
}
wait_mds_ost_sync () {
- # just because recovery is done doesn't mean we've finished
- # orphan cleanup. Wait for llogs to get synchronized.
- echo "Waiting for orphan cleanup..."
- # MAX value includes time needed for MDS-OST reconnection
- local MAX=$(( TIMEOUT * 2 ))
- local WAIT=0
- while [ $WAIT -lt $MAX ]; do
- local -a sync=($(do_nodes $(comma_list $(mdts_nodes)) \
- "$LCTL get_param -n osp.*osc*.old_sync_processed"))
- local con=1
- local i
- for ((i=0; i<${#sync[@]}; i++)); do
- [ ${sync[$i]} -eq 1 ] && continue
- # there is a not finished MDS-OST synchronization
- con=0
- break;
- done
- sleep 2 # increase waiting time and cover statfs cache
- [ ${con} -eq 1 ] && return 0
- echo "Waiting $WAIT secs for $facet mds-ost sync done."
- WAIT=$((WAIT + 2))
- done
- echo "$facet recovery not done in $MAX sec. $STATUS"
- return 1
+ # just because recovery is done doesn't mean we've finished
+ # orphan cleanup. Wait for llogs to get synchronized.
+ echo "Waiting for orphan cleanup..."
+ # MAX value includes time needed for MDS-OST reconnection
+ local MAX=$(( TIMEOUT * 2 ))
+ local WAIT=0
+ local new_wait=true
+ local list=$(comma_list $(mdts_nodes))
+ local cmd="$LCTL get_param -n osp.*osc*.old_sync_processed"
+ if ! do_facet $SINGLEMDS \
+ "$LCTL list_param osp.*osc*.old_sync_processed 2> /dev/null"
+ then
+ # old way, use mds_sync
+ new_wait=false
+ list=$(comma_list $(osts_nodes))
+ cmd="$LCTL get_param -n obdfilter.*.mds_sync"
+ fi
+ while [ $WAIT -lt $MAX ]; do
+ local -a sync=($(do_nodes $list "$cmd"))
+ local con=1
+ local i
+ for ((i=0; i<${#sync[@]}; i++)); do
+ if $new_wait; then
+ [ ${sync[$i]} -eq 1 ] && continue
+ else
+ [ ${sync[$i]} -eq 0 ] && continue
+ fi
+ # there is a not finished MDS-OST synchronization
+ con=0
+ break;
+ done
+ sleep 2 # increase waiting time and cover statfs cache
+ [ ${con} -eq 1 ] && return 0
+ echo "Waiting $WAIT secs for $facet mds-ost sync done."
+ WAIT=$((WAIT + 2))
+ done
+ echo "$facet recovery not done in $MAX sec. $STATUS"
+ return 1
}
wait_destroy_complete () {
}
fail() {
- facet_failover $* || error "failover: $?"
- clients_up || error "post-failover df: $?"
+ local facets=$1
+ local clients=${CLIENTS:-$HOSTNAME}
+
+ facet_failover $* || error "failover: $?"
+ wait_clients_import_state "$clients" "$facets" FULL
+ clients_up || error "post-failover df: $?"
}
fail_nodf() {
if ! remote_mds_nodsh; then
for num in $(seq $MDSCOUNT); do
DEVNAME=`mdsdevname $num`
- eval export MDSDEV${num}=$DEVNAME
init_facet_vars mds$num $DEVNAME $MDS_MOUNT_OPTS
done
fi
- eval export MGSDEV=$(mgsdevname)
combined_mgs_mds || init_facet_vars mgs $(mgsdevname) $MGS_MOUNT_OPTS
if ! remote_ost_nodsh; then
for num in $(seq $OSTCOUNT); do
DEVNAME=$(ostdevname $num)
- eval export OSTDEV${num}=$DEVNAME
init_facet_vars ost$num $DEVNAME $OST_MOUNT_OPTS
done
fi
}
is_empty_dir() {
- [ $(find $1 -maxdepth 1 -print | wc -l) = 1 ] && return 0
- return 1
+ [ $(find $1 -maxdepth 1 -print | wc -l) = 1 ] && return 0
+ return 1
}
# empty lustre filesystem may have empty directories lost+found and .lustre
is_empty_fs() {
- [ $(find $1 -maxdepth 1 -name lost+found -o -name .lustre -prune -o \
- -print | wc -l) = 1 ] || return 1
- [ ! -d $1/lost+found ] || is_empty_dir $1/lost+found && return 0
- [ ! -d $1/.lustre ] || is_empty_dir $1/.lustre && return 0
- return 1
+ [ $(find $1 -maxdepth 1 -name lost+found -o -name .lustre -prune -o \
+ -print | wc -l) = 1 ] || return 1
+ [ ! -d $1/lost+found ] || is_empty_dir $1/lost+found || return 1
+ [ ! -d $1/.lustre ] || is_empty_dir $1/.lustre || return 1
+ return 0
}
check_and_setup_lustre() {
local dev
if [ "$type" == ost ]; then
- devs=$(get_obdfilter_param $node "" mntdev)
+ devs=$(get_osd_param $node "" mntdev)
else
devs=$(do_node $node \
"lctl get_param -n osd-*.$FSNAME-M*.mntdev")
done
}
+# Run lfsck on server node if lfsck can't be found on client (LU-2571)
+run_lfsck_remote() {
+ local cmd="$LFSCK_BIN -c -l --mdsdb $MDSDB --ostdb $OSTDB_LIST $MOUNT"
+ local client=$1
+ local mounted=true
+ local rc=0
+
+ #Check if lustre is already mounted
+ do_rpc_nodes $client is_mounted $MOUNT || mounted=false
+ if ! $mounted; then
+ zconf_mount $client $MOUNT ||
+ error "failed to mount Lustre on $client"
+ fi
+ #Run lfsck
+ echo $cmd
+ do_node $node $cmd || rc=$?
+ #Umount if necessary
+ if ! $mounted; then
+ zconf_umount $client $MOUNT ||
+ error "failed to unmount Lustre on $client"
+ fi
+
+ [ $rc -le $FSCK_MAX_ERR ] ||
+ error "$cmd returned $rc, should be <= $FSCK_MAX_ERR"
+ echo "lfsck finished with rc=$rc"
+
+ return $rc
+}
+
run_lfsck() {
- local cmd="$LFSCK_BIN -c -l --mdsdb $MDSDB --ostdb $OSTDB_LIST $MOUNT"
- echo $cmd
- local rc=0
- eval $cmd || rc=$?
- [ $rc -le $FSCK_MAX_ERR ] || \
- error "$cmd returned $rc, should be <= $FSCK_MAX_ERR"
- echo "lfsck finished with rc=$rc"
+ local facets="client $SINGLEMDS"
+ local found=false
+ local facet
+ local node
+ local rc=0
- rm -rvf $MDSDB* $OSTDB* || true
- return 0
+ for facet in $facets; do
+ node=$(facet_active_host $facet)
+ if check_progs_installed $node $LFSCK_BIN; then
+ found=true
+ break
+ fi
+ done
+ ! $found && error "None of \"$facets\" supports lfsck"
+
+ run_lfsck_remote $node || rc=$?
+
+ rm -rvf $MDSDB* $OSTDB* || true
+ return $rc
}
check_and_cleanup_lustre() {
}
drop_update_reply() {
-# OBD_FAIL_MDS_OBJ_UPDATE_NET
+# OBD_FAIL_UPDATE_OBJ_NET_REP
local index=$1
shift 1
RC=0
- do_facet mds${index} lctl set_param fail_loc=0x188
+ do_facet mds${index} lctl set_param fail_loc=0x1701
do_facet client "$@" || RC=$?
do_facet mds${index} lctl set_param fail_loc=0
return $RC
[ -z "$rnodes" ] && return 0
- do_nodes "$rnodes" "rc=\\\$([ -f $C ] && echo \\\$(< $C) || echo 0);
+ local data
+ data=$(do_nodes "$rnodes" "rc=\\\$([ -f $C ] &&
+ echo \\\$(< $C) || echo 0);
if [ \\\$rc -ne 0 ]; then echo \\\$(hostname): \\\$rc; fi
- exit \\\$rc;"
+ exit \\\$rc")
+ local rc=$?
+ if [ -n "$data" ]; then
+ echo $data
+ return $rc
+ fi
+ return 0
}
# CMD: determine mds index where directory inode presents
}
get_clientosc_proc_path() {
- echo "${1}-osc-[^M]*"
+ echo "${1}-osc-*"
}
get_lustre_version () {
local CONN_STATE
local i=0
- CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2)
+ CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2 | uniq)
while [ "${CONN_STATE}" != "${expected}" ]; do
if [ "${expected}" == "DISCONN" ]; then
# for disconn we can check after proc entry is removed
error "can't put import for $CONN_PROC into ${expected} state after $i sec, have ${CONN_STATE}" && \
return 1
sleep 1
- CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2)
+ # Add uniq for multi-mount case
+ CONN_STATE=$($LCTL get_param -n $CONN_PROC 2>/dev/null | cut -f2 | uniq)
i=$(($i + 1))
done
done
}
+wait_import_state_mount() {
+ if ! is_mounted $MOUNT && ! is_mounted $MOUNT2; then
+ return 0
+ fi
+
+ wait_import_state $*
+}
+
# One client request could be timed out because server was not ready
# when request was sent by client.
# The request timeout calculation details :
local expected=$3
local ost=$(get_osc_import_name $facet $ost_facet)
- local param="osc.${ost}.ost_server_uuid"
+ local param="osc.${ost}.ost_server_uuid"
+ local i=0
# 1. wait the deadline of client 1st request (it could be skipped)
# 2. wait the deadline of client 2nd request
local maxtime=$(( 2 * $(request_timeout $facet)))
+ #During setup time, the osc might not be setup, it need wait
+ #until list_param can return valid value. And also if there
+ #are mulitple osc entries we should list all of them before
+ #go to wait.
+ local params=$($LCTL list_param $param || true)
+ while [ -z "$params" ]; do
+ if [ $i -ge $maxtime ]; then
+ echo "can't get $param by list_param in $maxtime secs"
+ if [[ $facet != client* ]]; then
+ echo "Go with $param directly"
+ params=$param
+ break
+ else
+ return 1
+ fi
+ fi
+ sleep 1
+ i=$((i + 1))
+ params=$($LCTL list_param $param || true)
+ done
+
if ! do_rpc_nodes "$(facet_host $facet)" \
- _wait_import_state $expected $param $maxtime; then
+ wait_import_state $expected "$params" $maxtime; then
error "import is not in ${expected} state"
return 1
fi
local params=$(expand_list $params $proc_path)
done
- if ! do_rpc_nodes "$list" wait_import_state $expected $params; then
+ if ! do_rpc_nodes "$list" wait_import_state_mount $expected $params; then
error "import is not in ${expected} state"
return 1
fi
echo "removing files from $mdtdev on $facet: $files"
if [ $(facet_fstype $facet) == ldiskfs ] &&
- ! do_facet $facet test -b ${!dev}; then
+ ! do_facet $facet test -b $mdtdev; then
opts=$(csa_add "$opts" -o loop)
fi
mount -t $(facet_fstype $facet) $opts $mdtdev $mntpt ||
echo "duplicating files on $mdtdev on $facet: $files"
mkdir -p $mntpt || return $?
if [ $(facet_fstype $facet) == ldiskfs ] &&
- ! do_facet $facet test -b ${!dev}; then
+ ! do_facet $facet test -b $mdtdev; then
opts=$(csa_add "$opts" -o loop)
fi
mount -t $(facet_fstype $facet) $opts $mdtdev $mntpt ||
local metadata=${TMP}/backup_restore.tgz
local opts=${MDS_MOUNT_OPTS}
local svc=${SINGLEMDS}_svc
+ local igif=$1
if ! ${rcmd} test -b ${devname}; then
opts=$(csa_add "$opts" -o loop)
${rcmd} rm -f $metaea $metadata
# step 3: mount dev
${rcmd} mount -t ldiskfs $opts $devname $mntpt || return 1
+ if [ ! -z $igif ]; then
+ # step 3.5: rm .lustre
+ ${rcmd} rm -rf $mntpt/ROOT/.lustre || return 1
+ fi
# step 4: backup metaea
echo "backup EA"
${rcmd} "cd $mntpt && getfattr -R -d -m '.*' -P . > $metaea && cd -" ||