# Constants used in more than one test script
export LOV_MAX_STRIPE_COUNT=2000
+ export DELETE_OLD_POOLS=${DELETE_OLD_POOLS:-false}
+ export KEEP_POOLS=${KEEP_POOLS:-false}
export MACHINEFILE=${MACHINEFILE:-$TMP/$(basename $0 .sh).machines}
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
get_lustre_env
# use localrecov to enable recovery for local clients, LU-12722
- [[ $MDS1_VERSION -lt $(version_code 2.13.52) ]] ||
+ [[ $MDS1_VERSION -lt $(version_code 2.13.52) ]] || {
export MDS_MOUNT_OPTS=${MDS_MOUNT_OPTS:-"-o localrecov"}
+ export MGS_MOUNT_OPTS=${MGS_MOUNT_OPTS:-"-o localrecov"}
+ }
+
[[ $OST1_VERSION -lt $(version_code 2.13.52) ]] ||
export OST_MOUNT_OPTS=${OST_MOUNT_OPTS:-"-o localrecov"}
}
echo -n $label
}
-mdsdevlabel() {
- local num=$1
- local device=$(mdsdevname $num)
- local label=$(devicelabel mds$num ${device} | grep -v "CMD: ")
- echo -n $label
-}
-
-ostdevlabel() {
- local num=$1
- local device=$(ostdevname $num)
- local label=$(devicelabel ost$num ${device} | grep -v "CMD: ")
- echo -n $label
-}
-
#
# Get the device of a facet.
#
}
# End recovery-scale functions
-# verify that lustre actually cleaned up properly
-cleanup_check() {
- VAR=$(lctl get_param -n catastrophe 2>&1)
- if [ $? = 0 ] ; then
- if [ $VAR != 0 ]; then
- error "LBUG/LASSERT detected"
- fi
- fi
- BUSY=$(dmesg | grep -i destruct || true)
- if [ -n "$BUSY" ]; then
- echo "$BUSY" 1>&2
- [ -e $TMP/debug ] && mv $TMP/debug $TMP/debug-busy.$(date +%s)
- exit 205
- fi
-
- check_mem_leak || exit 204
-
- [[ $($LCTL dl 2>/dev/null | wc -l) -gt 0 ]] && $LCTL dl &&
- echo "$TESTSUITE: lustre didn't clean up..." 1>&2 &&
- return 202 || true
-
- if module_loaded lnet || module_loaded libcfs; then
- echo "$TESTSUITE: modules still loaded..." 1>&2
- /sbin/lsmod 1>&2
- return 203
- fi
- return 0
-}
-
##
# wait for a command to return the expected result
#
}
wait_destroy_complete () {
- echo "Waiting for local destroys to complete"
+ echo "Waiting for MDT destroys to complete"
# MAX value shouldn't be big as this mean server responsiveness
# never increase this just to make test pass but investigate
# why it takes so long time
local MAX=5
local WAIT=0
+ local list=$(comma_list $(mdts_nodes))
while [ $WAIT -lt $MAX ]; do
- local -a RPCs=($($LCTL get_param -n osc.*.destroys_in_flight))
+ local -a RPCs=($(do_nodes $list $LCTL get_param -n osp.*.destroys_in_flight))
local con=1
local i
echo "Waiting ${WAIT}s for local destroys to complete"
WAIT=$((WAIT + 1))
done
- echo "Local destroys weren't done in $MAX sec."
+ echo "MDT destroys weren't done in $MAX sec."
return 1
}
done
}
-obd_name() {
- local facet=$1
-}
-
replay_barrier() {
local facet=$1
do_facet $facet "sync; sync; sync"
TIMEOUT=$(do_facet $SINGLEMDS "lctl get_param -n timeout")
log "Using TIMEOUT=$TIMEOUT"
+ # tune down to speed up testing on (usually) small setups
+ do_nodes $(comma_list $(nodes_list)) \
+ "echo 1 >/sys/module/mgc/parameters/mgc_requeue_timeout_min"
+
osc_ensure_active $SINGLEMDS $TIMEOUT
osc_ensure_active client $TIMEOUT
$LCTL set_param osc.*.idle_timeout=debug
echo $mounted' ' | grep -w -q $mntpt' '
}
-is_empty_dir() {
- [ $(find $1 -maxdepth 1 -print | wc -l) = 1 ] && return 0
- return 1
-}
+create_pools () {
+ local pool=$1
+ local ostsn=${2:-$OSTCOUNT}
+ local npools=${FS_NPOOLS:-$((OSTCOUNT / ostsn))}
+ local n
-# empty lustre filesystem may have empty directories lost+found and .lustre
-is_empty_fs() {
- # exclude .lustre & lost+found
- [ $(find $1 -maxdepth 1 -name lost+found -o -name .lustre -prune -o \
- -print | wc -l) = 1 ] || return 1
- [ ! -d $1/lost+found ] || is_empty_dir $1/lost+found || return 1
- if [ $(lustre_version_code $SINGLEMDS) -gt $(version_code 2.4.0) ]; then
- # exclude .lustre/fid (LU-2780)
- [ $(find $1/.lustre -maxdepth 1 -name fid -prune -o \
- -print | wc -l) = 1 ] || return 1
- else
- [ ! -d $1/.lustre ] || is_empty_dir $1/.lustre || return 1
- fi
- return 0
+ echo ostsn=$ostsn npools=$npools
+ if [[ $ostsn -gt $OSTCOUNT ]]; then
+ echo "request to use $ostsn OSTs in the pool, \
+ using max available OSTCOUNT=$OSTCOUNT"
+ ostsn=$OSTCOUNT
+ fi
+ for (( n=0; n < $npools; n++ )); do
+ p=${pool}$n
+ if ! $DELETE_OLD_POOLS; then
+ log "request to not delete old pools: $FSNAME.$p exist?"
+ if ! check_pool_not_exist $FSNAME.$p; then
+ echo "Using existing $FSNAME.$p"
+ $LCTL pool_list $FSNAME.$p
+ continue
+ fi
+ fi
+ create_pool $FSNAME.$p $KEEP_POOLS ||
+ error "create_pool $FSNAME.$p failed"
+
+ local first=$(( (n * ostsn) % OSTCOUNT ))
+ local last=$(( (first + ostsn - 1) % OSTCOUNT ))
+ if [[ $first -le $last ]]; then
+ pool_add_targets $p $first $last ||
+ error "pool_add_targets $p $first $last failed"
+ else
+ pool_add_targets $p $first $(( OSTCOUNT - 1 )) ||
+ error "pool_add_targets $p $first \
+ $(( OSTCOUNT - 1 )) failed"
+ pool_add_targets $p 0 $last ||
+ error "pool_add_targets $p 0 $last failed"
+ fi
+ done
}
check_and_setup_lustre() {
set_flavor_all $SEC
fi
+ if $DELETE_OLD_POOLS; then
+ destroy_all_pools
+ fi
+ if [[ -n "$FS_POOL" ]]; then
+ create_pools $FS_POOL $FS_POOL_NOSTS
+ fi
+
if [ "$ONLY" == "setup" ]; then
exit 0
fi
check_and_setup_lustre
}
-# Get all of the server target devices from a given server node and type.
-get_mnt_devs() {
- local node=$1
- local type=$2
- local devs
- local dev
-
- if [ "$type" == ost ]; then
- devs=$(get_osd_param $node "" mntdev)
- else
- devs=$(do_node $node $LCTL get_param -n osd-*.$FSNAME-M*.mntdev)
- fi
- for dev in $devs; do
- case $dev in
- *loop*) do_node $node "losetup $dev" | \
- sed -e "s/.*(//" -e "s/).*//" ;;
- *) echo $dev ;;
- esac
- done
-}
-
-# Get all of the server target devices.
-get_svr_devs() {
- local node
- local i
-
- # Master MDS parameters used by lfsck
- MDTNODE=$(facet_active_host $SINGLEMDS)
- MDTDEV=$(echo $(get_mnt_devs $MDTNODE mdt) | awk '{print $1}')
-
- # MDT devices
- i=0
- for node in $(mdts_nodes); do
- MDTDEVS[i]=$(get_mnt_devs $node mdt)
- i=$((i + 1))
- done
-
- # OST devices
- i=0
- for node in $(osts_nodes); do
- OSTDEVS[i]=$(get_mnt_devs $node ost)
- i=$((i + 1))
- done
-}
-
# Run e2fsck on MDT or OST device.
run_e2fsck() {
local node=$1
at_get $1 at_max
}
-at_min_get() {
- at_get $1 at_min
-}
-
at_max_set() {
local at_max=$1
shift
log_sub_test_end "SKIP" "0" "0" "$@"
}
-canonical_path() {
- (cd $(dirname $1); echo $PWD/$(basename $1))
-}
-
grant_from_clients() {
local nodes="$1"
echo -n $(facets_nodes $(get_facets OST))
}
-# Get all of the active AGT (HSM agent) nodes.
-agts_nodes () {
- echo -n $(facets_nodes $(get_facets AGT))
-}
-
# Get all of the client nodes and active server nodes.
nodes_list () {
local nodes=$HOSTNAME
echo $nodes | wc -w || true
}
-mixed_ost_devs () {
- local nodes=$(osts_nodes)
- local osscount=$(get_node_count "$nodes")
- [ ! "$OSTCOUNT" = "$osscount" ]
-}
-
mixed_mdt_devs () {
local nodes=$(mdts_nodes)
local mdtcount=$(get_node_count "$nodes")
rm -f $file
}
-setstripe_nfsserver () {
- local dir=$1
- local nfsexportdir=$2
- shift
- shift
-
- local -a nfsexport=($(awk '"'$dir'" ~ $2 && $3 ~ "nfs" && $2 != "/" \
- { print $1 }' /proc/mounts | cut -f 1 -d :))
-
- # check that only one nfs mounted
- [[ -z $nfsexport ]] && echo "$dir is not nfs mounted" && return 1
- (( ${#nfsexport[@]} == 1 )) ||
- error "several nfs mounts found for $dir: ${nfsexport[@]} !"
-
- do_nodev ${nfsexport[0]} lfs setstripe $nfsexportdir "$@"
-}
-
# Check and add a test group.
add_group() {
local group_id=$1
fi
}
-delayed_recovery_enabled () {
- local var=${SINGLEMDS}_svc
- do_facet $SINGLEMDS lctl get_param -n mdd.${!var}.stale_export_age > /dev/null 2>&1
-}
-
########################
convert_facet2label() {
echo "${1}-osc-[-0-9a-f]*"
}
-# If the 2.0 MDS was mounted on 1.8 device, then the OSC and LOV names
-# used by MDT would not be changed.
-# mdt lov: fsname-mdtlov
-# mdt osc: fsname-OSTXXXX-osc
-mds_on_old_device() {
- local mds=${1:-"$SINGLEMDS"}
-
- if [ $(lustre_version_code $mds) -gt $(version_code 1.9.0) ]; then
- do_facet $mds "lctl list_param osc.$FSNAME-OST*-osc \
- > /dev/null 2>&1" && return 0
- fi
- return 1
-}
-
get_mdtosc_proc_path() {
local mds_facet=$1
local ost_label=${2:-"*OST*"}
create_pool() {
local fsname=${1%%.*}
local poolname=${1##$fsname.}
+ local keep_pools=${2:-false}
stack_trap "destroy_test_pools $fsname" EXIT
do_facet mgs lctl pool_new $1
wait_update $HOSTNAME "lctl get_param -n lov.$fsname-*.pools.$poolname \
2>/dev/null || echo foo" "" || error "pool_new failed $1"
- add_pool_to_list $1
+ $keep_pools || add_pool_to_list $1
return $RC
}
local poolname=${1##$fsname.}
local listvar=${fsname}_CREATED_POOLS
- local temp=${listvar}=$(exclude_items_from_list ${!listvar} $poolname)
+ local temp=${listvar}=$(exclude_items_from_list "${!listvar}" $poolname)
eval export $temp
}
+# cleanup all pools exist on $FSNAME
+destroy_all_pools () {
+ local i
+ for i in $(list_pool $FSNAME); do
+ destroy_pool $i
+ done
+}
+
destroy_pool_int() {
local ost
local OSTS=$(list_pool $1)
local RC
- check_pool_not_exist $fsname.$poolname
- [[ $? -eq 0 ]] && return 0
+ check_pool_not_exist $fsname.$poolname && return 0 || true
destroy_pool_int $fsname.$poolname
RC=$?
llverfs $partial_arg $llverfs_opts $dir
}
-#Remove objects from OST
-remove_ost_objects() {
- local facet=$1
- local ostdev=$2
- local group=$3
- shift 3
- local objids="$@"
- local mntpt=$(facet_mntpt $facet)
- local opts=$OST_MOUNT_OPTS
- local i
- local rc
-
- echo "removing objects from $ostdev on $facet: $objids"
- if ! test -b $ostdev; then
- opts=$(csa_add "$opts" -o loop)
- fi
- mount -t $(facet_fstype $facet) $opts $ostdev $mntpt ||
- return $?
- rc=0
- for i in $objids; do
- rm $mntpt/O/$group/d$((i % 32))/$i || { rc=$?; break; }
- done
- umount -f $mntpt || return $?
- return $rc
-}
-
-#Remove files from MDT
-remove_mdt_files() {
- local facet=$1
- local mdtdev=$2
- shift 2
- local files="$@"
- local mntpt=$(facet_mntpt $facet)
- local opts=$MDS_MOUNT_OPTS
-
- echo "removing files from $mdtdev on $facet: $files"
- if [ $(facet_fstype $facet) == ldiskfs ] &&
- ! do_facet $facet test -b $mdtdev; then
- opts=$(csa_add "$opts" -o loop)
- fi
- mount -t $(facet_fstype $facet) $opts $mdtdev $mntpt ||
- return $?
- rc=0
- for f in $files; do
- rm $mntpt/ROOT/$f || { rc=$?; break; }
- done
- umount -f $mntpt || return $?
- return $rc
-}
-
-duplicate_mdt_files() {
- local facet=$1
- local mdtdev=$2
- shift 2
- local files="$@"
- local mntpt=$(facet_mntpt $facet)
- local opts=$MDS_MOUNT_OPTS
-
- echo "duplicating files on $mdtdev on $facet: $files"
- mkdir -p $mntpt || return $?
- if [ $(facet_fstype $facet) == ldiskfs ] &&
- ! do_facet $facet test -b $mdtdev; then
- opts=$(csa_add "$opts" -o loop)
- fi
- mount -t $(facet_fstype $facet) $opts $mdtdev $mntpt ||
- return $?
-
- do_umount() {
- trap 0
- popd > /dev/null
- rm $tmp
- umount -f $mntpt
- }
- trap do_umount EXIT
-
- tmp=$(mktemp $TMP/setfattr.XXXXXXXXXX)
- pushd $mntpt/ROOT > /dev/null || return $?
- rc=0
- for f in $files; do
- touch $f.bad || return $?
- getfattr -n trusted.lov $f | sed "s#$f#&.bad#" > $tmp
- rc=${PIPESTATUS[0]}
- [ $rc -eq 0 ] || return $rc
- setfattr --restore $tmp || return $?
- done
- do_umount
-}
-
run_sgpdd () {
local devs=${1//,/ }
shift
echo -n ${count:-0}
}
-# Get the block size of the filesystem.
-get_block_size() {
- local facet=$1
- local device=$2
- local size
-
- [ -z "$CLIENTONLY" ] && size=$(do_facet $facet "$DUMPE2FS -h $device 2>&1" |
- awk '/^Block size:/ {print $3}')
- echo -n ${size:-0}
-}
-
# Check whether the "ea_inode" feature is enabled or not, to allow
# ldiskfs xattrs over one block in size. Allow both the historical
# Lustre feature name (large_xattr) and the upstream name (ea_inode).
is_mounted $MOUNT || setupall
rm -rf $DIR/[df][0-9]* || error "Fail to cleanup the env!"
- mkdir $DIR/$tdir || error "Fail to mkdir $DIR/$tdir."
+ mkdir_on_mdt0 $DIR/$tdir || error "Fail to mkdir $DIR/$tdir."
for idx in $(seq $MDSCOUNT); do
local name="MDT$(printf '%04x' $((idx - 1)))"
rm -rf $MOUNT/.lustre/lost+found/$name/*
error "$mdt: changelog_mask=+hsm failed: $?"
local cl_user
- cl_user=$(do_facet $facet \
- $LCTL --device $mdt changelog_register -n) ||
+ cl_user=$(do_facet $facet $LCTL --device $mdt \
+ changelog_register -n $@) ||
error "$mdt: register changelog user failed: $?"
stack_trap "__changelog_deregister $facet $cl_user" EXIT
# so reorder to get same order than in changelog_register()
local cl_facets=$(echo "${!CL_USERS[@]}" | tr " " "\n" | sort |
tr "\n" " ")
+ local cl_user
for facet in $cl_facets; do
for cl_user in ${CL_USERS[$facet]}; do
local saved_debug=$($LCTL get_param -n debug)
local list=$(comma_list $(all_nodes))
- do_nodes $list $LCTL set_param debug=0
+ do_nodes $list $LCTL set_param -n debug=0
}
$LUSTRE/tests/createmany $*
local rc=$?
(( count > 100 )) &&
- do_nodes $list "$LCTL set_param debug=\\\"$saved_debug\\\""
+ do_nodes $list "$LCTL set_param -n debug=\\\"$saved_debug\\\""
return $rc
}
local saved_debug=$($LCTL get_param -n debug)
local list=$(comma_list $(all_nodes))
- do_nodes $list $LCTL set_param debug=0
+ do_nodes $list $LCTL set_param -n debug=0
}
$LUSTRE/tests/unlinkmany $*
local rc=$?
(( count > 100 )) &&
- do_nodes $list "$LCTL set_param debug=\\\"$saved_debug\\\""
+ do_nodes $list "$LCTL set_param -n debug=\\\"$saved_debug\\\""
return $rc
}
check_set_fallocate || skip "need at least 2.13.57 for fallocate"
}
+function disable_opencache()
+{
+ local state=$($LCTL get_param -n "llite.*.opencache_threshold_count" | head -1)
+
+ test -z "${saved_OPENCACHE_value}" &&
+ export saved_OPENCACHE_value="$state"
+
+ [[ "$state" = "off" ]] && return
+
+ $LCTL set_param -n "llite.*.opencache_threshold_count"=off
+}
+
+function set_opencache()
+{
+ local newvalue="$1"
+ local state=$($LCTL get_param -n "llite.*.opencache_threshold_count")
+
+ [[ -n "$newvalue" ]] || return
+
+ [[ -n "${saved_OPENCACHE_value}" ]] ||
+ export saved_OPENCACHE_value="$state"
+
+ $LCTL set_param -n "llite.*.opencache_threshold_count"=$newvalue
+}
+
+
+
+function restore_opencache()
+{
+ [[ -z "${saved_OPENCACHE_value}" ]] ||
+ $LCTL set_param -n "llite.*.opencache_threshold_count"=${saved_OPENCACHE_value}
+}
+
+# LU-13417: XXX lots of tests assume the directory to be created under MDT0,
+# using this function to create directory under MDT0 explicitly.
+# Don't use it in new tests, and remove it from old tests.
+mkdir_on_mdt0() {
+ $LFS mkdir -i 0 -c 1 $*
+}
+
+# Wait for nodemap synchronization
+wait_nm_sync() {
+ local nodemap_name=$1
+ local key=$2
+ local value=$3
+ local opt=$4
+ local proc_param
+ local is_active=$(do_facet mgs $LCTL get_param -n nodemap.active)
+ local max_retries=20
+ local is_sync
+ local out1=""
+ local out2
+ local mgs_ip=$(host_nids_address $mgs_HOST $NETTYPE | cut -d' ' -f1)
+ local i
+
+ if [ "$nodemap_name" == "active" ]; then
+ proc_param="active"
+ elif [ -z "$key" ]; then
+ proc_param=${nodemap_name}
+ else
+ proc_param="${nodemap_name}.${key}"
+ fi
+ if [ "$opt" == "inactive" ]; then
+ # check nm sync even if nodemap is not activated
+ is_active=1
+ opt=""
+ fi
+ (( is_active == 0 )) && [ "$proc_param" != "active" ] && return
+
+ if [ -z "$value" ]; then
+ out1=$(do_facet mgs $LCTL get_param $opt \
+ nodemap.${proc_param} 2>/dev/null)
+ echo "On MGS ${mgs_ip}, ${proc_param} = $out1"
+ else
+ out1=$value;
+ fi
+
+ # if servers run on the same node, it is impossible to tell if they get
+ # synced with the mgs, so just wait an arbitrary 10 seconds
+ if [ $(facet_active_host mgs) == $(facet_active_host mds) ] &&
+ [ $(facet_active_host mgs) == $(facet_active_host ost1) ]; then
+ echo "waiting 10 secs for sync"
+ sleep 10
+ return
+ fi
+
+ # wait up to 10 seconds for other servers to sync with mgs
+ for i in $(seq 1 10); do
+ for node in $(all_server_nodes); do
+ local node_ip=$(host_nids_address $node $NETTYPE |
+ cut -d' ' -f1)
+
+ is_sync=true
+ if [ -z "$value" ]; then
+ [ $node_ip == $mgs_ip ] && continue
+ fi
+
+ out2=$(do_node $node_ip $LCTL get_param $opt \
+ nodemap.$proc_param 2>/dev/null)
+ echo "On $node ${node_ip}, ${proc_param} = $out2"
+ [ "$out1" != "$out2" ] && is_sync=false && break
+ done
+ $is_sync && break
+ sleep 1
+ done
+ if ! $is_sync; then
+ echo MGS
+ echo $out1
+ echo OTHER - IP: $node_ip
+ echo $out2
+ error "mgs and $nodemap_name ${key} mismatch, $i attempts"
+ fi
+ echo "waited $((i - 1)) seconds for sync"
+}
git://git.whamcloud.com / fs / lustre-release.git / blobdiff