export MDS1_VERSION=$(lustre_version_code mds1)
export OST1_VERSION=$(lustre_version_code ost1)
export CLIENT_VERSION=$(lustre_version_code client)
+
+ # Prefer using "mds1" directly instead of SINGLEMDS.
+ # Keep this for compat until it is removed from scripts.
+ export SINGLEMDS=${SINGLEMDS:-mds1}
}
init_test_env() {
echo "unloading modules on: '$list'"
do_rpc_nodes "$list" $LUSTRE_RMMOD ldiskfs
do_rpc_nodes "$list" check_mem_leak
- do_rpc_nodes "$list" "rm /etc/udev/rules.d/99-lustre-test.rules"
+ do_rpc_nodes "$list" "rm -f /etc/udev/rules.d/99-lustre-test.rules"
do_rpc_nodes "$list" "udevadm control --reload-rules"
do_rpc_nodes "$list" "udevadm trigger"
fi
virt=$(dmidecode -s system-product-name | awk '{print $1}')
case $virt in
- VMware|KVM|VirtualBox|Parallels)
+ VMware|KVM|VirtualBox|Parallels|Bochs)
echo $virt | tr '[A-Z]' '[a-z]' ;;
*) ;;
esac
}
zconf_umount() {
- local client=$1
- local mnt=$2
- local force
- local busy
- local need_kill
-
- [ "$3" ] && force=-f
- local running=$(do_node $client "grep -c $mnt' ' /proc/mounts") || true
- if [ $running -ne 0 ]; then
- echo "Stopping client $client $mnt (opts:$force)"
- do_node $client lsof -t $mnt || need_kill=no
- if [ "x$force" != "x" -a "x$need_kill" != "xno" ]; then
- pids=$(do_node $client lsof -t $mnt | sort -u);
- if [ -n $pids ]; then
- do_node $client kill -9 $pids || true
- fi
- fi
-
- busy=$(do_node $client "umount $force $mnt 2>&1" | grep -c "busy") || true
- if [ $busy -ne 0 ] ; then
- echo "$mnt is still busy, wait one second" && sleep 1
- do_node $client umount $force $mnt
- fi
- fi
-}
-
-# Mount the file system on the MGS
-mount_mgs_client() {
- do_facet mgs "mkdir -p $MOUNT"
- zconf_mount $mgs_HOST $MOUNT $MOUNT_OPTS ||
- error "unable to mount $MOUNT on MGS"
-}
+ local client=$1
+ local mnt=$2
+ local force
+ local busy
+ local need_kill
+ local running=$(do_node $client "grep -c $mnt' ' /proc/mounts") || true
+
+ [ "$3" ] && force=-f
+ [ $running -eq 0 ] && return 0
+
+ echo "Stopping client $client $mnt (opts:$force)"
+ do_node $client lsof -t $mnt || need_kill=no
+ if [ "x$force" != "x" ] && [ "x$need_kill" != "xno" ]; then
+ pids=$(do_node $client lsof -t $mnt | sort -u);
+ if [ -n "$pids" ]; then
+ do_node $client kill -9 $pids || true
+ fi
+ fi
-# Unmount the file system on the MGS
-umount_mgs_client() {
- zconf_umount $mgs_HOST $MOUNT
- do_facet mgs "rm -rf $MOUNT"
+ busy=$(do_node $client "umount $force $mnt 2>&1" | grep -c "busy") ||
+ true
+ if [ $busy -ne 0 ] ; then
+ echo "$mnt is still busy, wait one second" && sleep 1
+ do_node $client umount $force $mnt
+ fi
}
# nodes is comma list
fi
}
+fill_ost() {
+ local filename=$1
+ local ost_idx=$2
+ local lwm=$3 #low watermark
+ local size_mb #how many MB should we write to pass watermark
+ local ost_name=$(ostname_from_index $ost_idx)
+
+ free_kb=$($LFS df $MOUNT | awk "/$ost_name/ { print \$4 }")
+ size_mb=0
+ if (( $free_kb / 1024 > lwm )); then
+ size_mb=$((free_kb / 1024 - lwm))
+ fi
+ #If 10% of free space cross low watermark use it
+ if (( $free_kb / 10240 > size_mb )); then
+ size_mb=$((free_kb / 10240))
+ else
+ #At least we need to store 1.1 of difference between
+ #free space and low watermark
+ size_mb=$((size_mb + size_mb / 10))
+ fi
+ if (( lwm <= $free_kb / 1024 )) ||
+ [ ! -f $DIR/${filename}.fill_ost$ost_idx ]; then
+ $LFS setstripe -i $ost_idx -c1 $DIR/${filename}.fill_ost$ost_idx
+ dd if=/dev/zero of=$DIR/${filename}.fill_ost$ost_idx bs=1M \
+ count=$size_mb oflag=append conv=notrunc
+ fi
+
+ sleep_maxage
+
+ free_kb=$($LFS df $MOUNT | awk "/$ost_name/ { print \$4 }")
+ echo "OST still has $((free_kb / 1024)) MB free"
+}
+
+# This checks only the primary MDS
+ost_watermarks_get() {
+ local ost_idx=$1
+ local ost_name=$(ostname_from_index $ost_idx)
+ local mdtosc_proc=$(get_mdtosc_proc_path $SINGLEMDS $ost_name)
+
+ local hwm=$(do_facet $SINGLEMDS $LCTL get_param -n \
+ osp.$mdtosc_proc.reserved_mb_high)
+ local lwm=$(do_facet $SINGLEMDS $LCTL get_param -n \
+ osp.$mdtosc_proc.reserved_mb_low)
+
+ echo "$lwm $hwm"
+}
+
+# Note that we set watermarks on all MDSes (necessary for striped dirs)
+ost_watermarks_set() {
+ local ost_idx=$1
+ local lwm=$2
+ local hwm=$3
+ local ost_name=$(ostname_from_index $ost_idx)
+ local facets=$(get_facets MDS)
+
+ do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param -n \
+ osp.*$ost_name*.reserved_mb_low=$lwm \
+ osp.*$ost_name*.reserved_mb_high=$hwm > /dev/null
+
+ # sleep to ensure we see the change
+ sleep_maxage
+}
+
+ost_watermarks_set_low_space() {
+ local ost_idx=$1
+ local wms=$(ost_watermarks_get $ost_idx)
+ local ost_name=$(ostname_from_index $ost_idx)
+
+ local old_lwm=$(echo $wms | awk '{ print $1 }')
+ local old_hwm=$(echo $wms | awk '{ print $2 }')
+
+ local blocks=$($LFS df $MOUNT | awk "/$ost_name/ { print \$4 }")
+ # minimal extension size is 64M
+ local new_lwm=50
+ if (( $blocks / 1024 > 50 )); then
+ new_lwm=$((blocks / 1024 - 50))
+ fi
+ local new_hwm=$((new_lwm + 5))
+
+ ost_watermarks_set $ost_idx $new_lwm $new_hwm
+ echo "watermarks: $old_lwm $old_hwm $new_lwm $new_hwm"
+}
+
+# Set watermarks to ~current available space & then write data to fill it
+# Note OST is not *actually* full after this, it just reports ENOSPC in the
+# internal statfs used by the stripe allocator
+#
+# first parameter is the filename-prefix, which must get under t-f cleanup
+# requirements (rm -rf $DIR/[Rdfs][0-9]*), i.e. $tfile work fine
+ost_watermarks_set_enospc() {
+ local filename=$1
+ local ost_idx=$2
+ # on the mdt's osc
+ local ost_name=$(ostname_from_index $ost_idx)
+ local facets=$(get_facets MDS)
+ local wms
+ local MDS
+
+ for MDS in ${facets//,/ }; do
+ local mdtosc_proc=$(get_mdtosc_proc_path $MDS $ost_name)
+
+ do_facet $MDS $LCTL get_param -n \
+ osp.$mdtosc_proc.reserved_mb_high ||
+ skip "remote MDS does not support reserved_mb_high"
+ done
+
+ wms=$(ost_watermarks_set_low_space $ost_idx)
+ local new_lwm=$(echo $wms | awk '{ print $4 }')
+ fill_ost $filename $ost_idx $new_lwm
+ #First enospc could execute orphan deletion so repeat
+ fill_ost $filename $ost_idx $new_lwm
+ echo $wms
+}
+
+ost_watermarks_enospc_delete_files() {
+ local filename=$1
+ local ost_idx=$2
+
+ rm -f $DIR/${filename}.fill_ost$ost_idx
+
+ wait_delete_completed
+ wait_mds_ost_sync
+}
+
+# clean up from "ost_watermarks_set_enospc"
+ost_watermarks_clear_enospc() {
+ local filename=$1
+ local ost_idx=$2
+ local old_lwm=$4
+ local old_hwm=$5
+
+ ost_watermarks_enospc_delete_files $filename $ost_idx
+ ost_watermarks_set $ost_idx $old_lwm $old_hwm
+ echo "set OST$ost_idx lwm back to $old_lwm, hwm back to $old_hwm"
+}
+
wait_delete_completed_mds() {
local max_wait=${1:-20}
local mds2sync=""
change_active ${affecteds[index]}
wait_for_facet ${affecteds[index]}
+ if $GSS_SK; then
+ init_gss
+ init_facets_vars_simple
+ fi
# start mgs first if it is affected
if ! combined_mgs_mds &&
list_member ${affecteds[index]} mgs; then
affected=$(exclude_items_from_list ${affecteds[index]} mgs)
echo mount facets: ${affecteds[index]}
mount_facets ${affecteds[index]}
+ if $GSS_SK; then
+ do_nodes $(comma_list $(all_nodes)) \
+ "keyctl show | grep lustre | cut -c1-11 |
+ sed -e 's/ //g;' |
+ xargs -IX keyctl setperm X 0x3f3f3f3f"
+ fi
done
}
local facets=$1
local clients=${CLIENTS:-$HOSTNAME}
+ SK_NO_KEY_save=$SK_NO_KEY
+ if $GSS_SK; then
+ export SK_NO_KEY=false
+ fi
facet_failover $* || error "failover: $?"
+ export SK_NO_KEY=$SK_NO_KEY_save
# to initiate all OSC idling connections
clients_up
wait_clients_import_state "$clients" "$facets" "\(FULL\|IDLE\)"
echo -n "$1" | tr '[:lower:]' '[:upper:]'
}
+squash_opt() {
+ local var="$*"
+ local other=""
+ local opt_o=""
+ local opt_e=""
+ local first_e=0
+ local first_o=0
+ local take=""
+
+ var=$(echo "$var" | sed -e 's/,\( \)*/,/g')
+ for i in $(echo "$var"); do
+ if [ "$i" == "-O" ]; then
+ take="o";
+ first_o=$(($first_o + 1))
+ continue;
+ fi
+ if [ "$i" == "-E" ]; then
+ take="e";
+ first_e=$(($first_e + 1 ))
+ continue;
+ fi
+ case $take in
+ "o")
+ [ $first_o -gt 1 ] && opt_o+=",";
+ opt_o+="$i";
+ ;;
+ "e")
+ [ $first_e -gt 1 ] && opt_e+=",";
+ opt_e+="$i";
+ ;;
+ *)
+ other+=" $i";
+ ;;
+ esac
+ take=""
+ done
+
+ echo -n "$other"
+ [ -n "$opt_o" ] && echo " -O $opt_o"
+ [ -n "$opt_e" ] && echo " -E $opt_e"
+}
+
mkfs_opts() {
local facet=$1
local dev=$2
opts+=${L_GETIDENTITY:+" --param=mdt.identity_upcall=$L_GETIDENTITY"}
if [ $fstype == ldiskfs ]; then
- fs_mkfs_opts+="-O ea_inode"
+ fs_mkfs_opts+="-O ea_inode,large_dir"
var=${facet}_JRN
if [ -n "${!var}" ]; then
var=${type}_FS_MKFS_OPTS
fs_mkfs_opts+=${!var:+" ${!var}"}
+ [ $fstype == ldiskfs ] && fs_mkfs_opts=$(squash_opt $fs_mkfs_opts)
+
if [ -n "${fs_mkfs_opts## }" ]; then
opts+=" --mkfsoptions=\\\"${fs_mkfs_opts## }\\\""
fi
fi
}
+init_facets_vars_simple () {
+ local devname
+
+ if ! remote_mds_nodsh; then
+ for num in $(seq $MDSCOUNT); do
+ devname=$(mdsdevname $num)
+ eval export mds${num}_dev=${devname}
+ eval export mds${num}_opt=\"${MDS_MOUNT_OPTS}\"
+ done
+ fi
+
+ if ! combined_mgs_mds ; then
+ eval export mgs_dev=$(mgsdevname)
+ eval export mgs_opt=\"${MGS_MOUNT_OPTS}\"
+ fi
+
+ if ! remote_ost_nodsh; then
+ for num in $(seq $OSTCOUNT); do
+ devname=$(ostdevname $num)
+ eval export ost${num}_dev=${devname}
+ eval export ost${num}_opt=\"${OST_MOUNT_OPTS}\"
+ done
+ fi
+}
+
osc_ensure_active () {
local facet=$1
local timeout=$2
TIMEOUT=$(lctl get_param -n timeout)
TIMEOUT=${TIMEOUT:-20}
- if [ -n $arg1 ]; then
+ if [ -n "$arg1" ]; then
[ "$arg1" = "server_only" ] && return
fi
# usage: stack_trap arg sigspec
#
# stack_trap() behaves like bash's built-in trap, except that it "stacks" the
-# command ``arg`` on top of previously defined commands for ``sigspec`` instead
+# command "arg" on top of previously defined commands for "sigspec" instead
# of overwriting them.
# stacked traps are executed in reverse order of their registration
#
stack_trap()
{
local arg="$1"
- local sigspec="$2"
+ local sigspec="${2:-EXIT}"
# Use "trap -p" to get the quoting right
local old_trap="$(trap -p "$sigspec")"
########################
convert_facet2label() {
- local facet=$1
+ local facet=$1
- if [ x$facet = xost ]; then
- facet=ost1
- fi
+ if [ x$facet = xost ]; then
+ facet=ost1
+ elif [ x$facet = xmgs ] && combined_mgs_mds ; then
+ facet=mds1
+ fi
- local varsvc=${facet}_svc
+ local varsvc=${facet}_svc
- if [ -n ${!varsvc} ]; then
- echo ${!varsvc}
- else
- error "No lablel for $facet!"
- fi
+ if [ -n ${!varsvc} ]; then
+ echo ${!varsvc}
+ else
+ error "No label for $facet!"
+ fi
}
get_clientosc_proc_path() {
for ost in $OSTS; do
do_facet mgs lctl pool_remove $1 $ost
done
+ wait_update_facet $SINGLEMDS "lctl pool_list $1 | wc -l" "1" ||
+ error "MDS: pool_list $1 failed"
do_facet mgs lctl pool_destroy $1
}
[ $(lustre_version_code $SINGLEMDS) -le \
$(version_code 2.10.53) ] && return 1
- do_fact mds1 $ZPOOL upgrade -v |
+ do_facet mds1 $ZPOOL upgrade -v |
grep project_quota && return 0
fi
wait ${!do_node_pid}
}
+
+sleep_maxage() {
+ local delay=$(do_facet $SINGLEMDS lctl get_param -n lo[vd].*.qos_maxage |
+ awk '{ print $1 * 2; exit; }')
+ sleep $delay
+}
+
+check_component_count() {
+ local comp_cnt=$($LFS getstripe --component-count $1)
+ [ $comp_cnt -eq $2 ] || error "$1, component count $comp_cnt != $2"
+}
+
+# Verify there are no init components with "extension" flag
+verify_no_init_extension() {
+ local flg_opts="--component-flags init,extension"
+ local found=$($LFS find $flg_opts $1 | wc -l)
+ [ $found -eq 0 ] || error "$1 has component with initialized extension"
+}
+
+# Verify there is at least one component starting at 0
+verify_comp_at_zero() {
+ flg_opts="--component-flags init"
+ found=$($LFS find --component-start 0M $flg_opts $1 | wc -l)
+ [ $found -eq 1 ] ||
+ error "No component starting at zero(!)"
+}
+
+#TODO: This version is a placeholder, to be replaced before final commit
+SEL_VER="2.12.52"
+
+sel_layout_sanity() {
+ local file=$1
+ local comp_cnt=$2
+
+ verify_no_init_extension $file
+ verify_comp_at_zero $file
+ check_component_count $file $comp_cnt
+}
+
git://git.whamcloud.com / fs / lustre-release.git / blobdiff