X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Ftest-framework.sh;h=b85a480285039b683970756561dd9a733dc12a0f;hp=146746ca70e10c15ef423eaa963392a67a18344a;hb=c52d48bf7838c8d9a886a8865017e150e294d91c;hpb=3172a9e96e63f70c306d03336cc6407659502b9c diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 146746c..b85a480 100644 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -5,6 +5,7 @@ trap 'print_summary && touch $TF_FAIL && \ set -e #set -x +export EJOURNAL=${EJOURNAL:-""} export REFORMAT=${REFORMAT:-""} export WRITECONF=${WRITECONF:-""} export VERBOSE=${VERBOSE:-false} @@ -14,6 +15,8 @@ export GSS_KRB5=false export GSS_PIPEFS=false export IDENTITY_UPCALL=default export QUOTA_AUTO=1 +export JOBSTATS_AUTO=${JOBSTATS_AUTO:-1} +export JOBID_VAR=${JOBID_VAR:-"procname_uid"} # LOAD_LLOOP: LU-409: only load llite_lloop module if kernel < 2.6.32 or # LOAD_LLOOP is true. LOAD_LLOOP is false by default. @@ -26,6 +29,8 @@ LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} . $LUSTRE/tests/functions.sh . $LUSTRE/tests/yaml.sh +export LD_LIBRARY_PATH=${LUSTRE}/utils:${LD_LIBRARY_PATH} + LUSTRE_TESTS_CFG_DIR=${LUSTRE_TESTS_CFG_DIR:-${LUSTRE}/tests/cfg} EXCEPT_LIST_FILE=${EXCEPT_LIST_FILE:-${LUSTRE_TESTS_CFG_DIR}/tests-to-skip.sh} @@ -185,11 +190,11 @@ init_test_env() { fi fi export LL_DECODE_FILTER_FID=${LL_DECODE_FILTER_FID:-"$LUSTRE/utils/ll_decode_filter_fid"} - [ ! -f "$LL_DECODE_FILTER_FID" ] && export LL_DECODE_FILTER_FID=$(which ll_decode_filter_fid) + [ ! -f "$LL_DECODE_FILTER_FID" ] && export LL_DECODE_FILTER_FID="ll_decode_filter_fid" export MKFS=${MKFS:-"$LUSTRE/utils/mkfs.lustre"} - [ ! -f "$MKFS" ] && export MKFS=$(which mkfs.lustre) + [ ! -f "$MKFS" ] && export MKFS="mkfs.lustre" export TUNEFS=${TUNEFS:-"$LUSTRE/utils/tunefs.lustre"} - [ ! -f "$TUNEFS" ] && export TUNEFS=$(which tunefs.lustre) + [ ! -f "$TUNEFS" ] && export TUNEFS="tunefs.lustre" export CHECKSTAT="${CHECKSTAT:-"checkstat -v"} " export LUSTRE_RMMOD=${LUSTRE_RMMOD:-$LUSTRE/scripts/lustre_rmmod} [ ! -f "$LUSTRE_RMMOD" ] && @@ -265,6 +270,20 @@ init_test_env() { rm -f $TMP/*active } +check_cpt_number() { + local facet=$1 + local ncpts + + ncpts=$(do_facet $facet "lctl get_param -n " \ + "cpu_partition_table 2>/dev/null| wc -l" || echo 1) + + if [ $ncpts -eq 0 ]; then + echo "1" + else + echo $ncpts + fi +} + version_code() { # split arguments like "1.8.6-wc3" into "1", "8", "6", "wc3" eval set -- $(tr "[:punct:]" " " <<< $*) @@ -356,6 +375,7 @@ llite_lloop_enabled() { } load_modules_local() { + [ $(facet_fstype ost1) == "zfs" ] && export USE_OFD=yes if [ "$USE_OFD" == yes ]; then if module_loaded obdfilter; then if ! $LUSTRE_RMMOD ldiskfs; then @@ -378,8 +398,39 @@ load_modules_local() { return 0 fi - echo Loading modules from $LUSTRE - load_module ../libcfs/libcfs/libcfs + echo Loading modules from $LUSTRE + + local ncpus + + if [ -f /sys/devices/system/cpu/online ]; then + ncpus=$(($(cut -d "-" -f 2 /sys/devices/system/cpu/online) + 1)) + echo "detected $ncpus online CPUs by sysfs" + else + ncpus=$(getconf _NPROCESSORS_CONF 2>/dev/null) + local rc=$? + if [ $rc -eq 0 ]; then + echo "detected $ncpus online CPUs by getconf" + else + echo "Can't detect number of CPUs" + ncpus=1 + fi + fi + + local ncpts=0 + # if there is only one CPU core, libcfs can only create one partition + # if there is more than 4 CPU cores, libcfs should create multiple CPU + # partitions. So we just force libcfs to create 2 partitions for + # system with 2 or 4 cores + if [ $ncpus -le 4 ] && [ $ncpus -gt 1 ]; then + # force to enable multiple CPU partitions + echo "Force libcfs to create 2 CPU partitions" + ncpts=2 + else + echo "libcfs will create CPU partition based on online CPUs" + fi + + load_module ../libcfs/libcfs/libcfs cpu_npartitions=$ncpts + [ "$PTLDEBUG" ] && lctl set_param debug="$PTLDEBUG" [ "$SUBSYSTEM" ] && lctl set_param subsystem_debug="${SUBSYSTEM# }" load_module ../lnet/lnet/lnet @@ -396,30 +447,41 @@ load_modules_local() { load_module osc/osc load_module lov/lov load_module mgc/mgc + load_module obdecho/obdecho if ! client_only; then SYMLIST=/proc/kallsyms grep -q crc16 $SYMLIST || { modprobe crc16 2>/dev/null || true; } grep -q -w jbd $SYMLIST || { modprobe jbd 2>/dev/null || true; } grep -q -w jbd2 $SYMLIST || { modprobe jbd2 2>/dev/null || true; } - if [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then - grep -q exportfs_decode_fh $SYMLIST || - { modprobe exportfs 2> /dev/null || true; } - load_module ../ldiskfs/ldiskfs/ldiskfs - fi [ "$LQUOTA" != "no" ] && load_module quota/lquota $LQUOTAOPTS + if [[ $(node_fstypes $HOSTNAME) == *zfs* ]]; then + modprobe zfs + load_module osd-zfs/osd_zfs + fi load_module mgs/mgs load_module mds/mds load_module mdd/mdd + if [[ $(node_fstypes $HOSTNAME) == *ldiskfs* ]]; then + # + # This block shall be moved up beside osd-zfs as soon + # as osd-ldiskfs stops using mdd symbols. + # + grep -q exportfs_decode_fh $SYMLIST || + { modprobe exportfs 2> /dev/null || true; } + load_module ../ldiskfs/ldiskfs/ldiskfs + load_module lvfs/fsfilt_ldiskfs + load_module osd-ldiskfs/osd_ldiskfs + fi load_module mdt/mdt - load_module lvfs/fsfilt_ldiskfs - load_module cmm/cmm - load_module osd-ldiskfs/osd_ldiskfs load_module ost/ost + load_module lod/lod + load_module osp/osp if [ "$USE_OFD" == yes ]; then load_module ofd/ofd else load_module obdfilter/obdfilter fi + load_module osp/osp fi @@ -694,6 +756,34 @@ ostdevlabel() { echo -n $label } +# +# This and set_obdfilter_param() shall be used to access OSD parameters +# once existed under "obdfilter": +# +# mntdev +# stats +# read_cache_enable +# writethrough_cache_enable +# +get_obdfilter_param() { + local nodes=$1 + local device=${2:-$FSNAME-OST*} + local name=$3 + + do_nodes $nodes "$LCTL get_param -n obdfilter.$device.$name \ + osd-*.$device.$name 2>&1" | grep -v 'Found no match' +} + +set_obdfilter_param() { + local nodes=$1 + local device=${2:-$FSNAME-OST*} + local name=$3 + local value=$4 + + do_nodes $nodes "$LCTL set_param -n obdfilter.$device.$name=$value \ + osd-*.$device.$name=$value 2>&1" | grep -v 'Found no match' +} + set_debug_size () { local dz=${1:-$DEBUG_SIZE} @@ -840,6 +930,39 @@ start() { return $RC } +# +# When a ZFS OSD is made read-only by replay_barrier(), its pool is "freezed". +# Because stopping corresponding target may not clear this in-memory state, we +# need to zap the pool from memory by exporting and reimporting the pool. +# +# Although the uberblocks are not updated when a pool is freezed, transactions +# are still written to the disks. Modified blocks may be cached in memory when +# tests try reading them back. The export-and-reimport process also evicts any +# cached pool data from memory to provide the correct "data loss" semantics. +# +refresh_disk() { + local facet=$1 + local fstype=$(facet_fstype $facet) + local _dev + local dev + local poolname + + if [ "${fstype}" == "zfs" ]; then + _dev=$(facet_active $facet)_dev + dev=${!_dev} # expand _dev to its value, e.g. ${mds1_dev} + poolname="${dev%%/*}" # poolname is string before "/" + + if [ "${poolname}" == "" ]; then + echo "invalid dataset name: $dev" + return + fi + do_facet $facet "cp /etc/zfs/zpool.cache /tmp/zpool.cache.back" + do_facet $facet "$ZPOOL export ${poolname}" + do_facet $facet "$ZPOOL import -f -c /tmp/zpool.cache.back \ + ${poolname}" + fi +} + stop() { local running local facet=$1 @@ -861,6 +984,10 @@ stop() { # save quota version (both administrative and operational quotas) # add an additional parameter if mountpoint is ever different from $MOUNT +# +# XXX This function is kept for interoperability with old server (< 2.3.50), +# it should be removed whenever we drop the interoperability for such +# server. quota_save_version() { local fsname=${2:-$FSNAME} local spec=$1 @@ -880,7 +1007,11 @@ quota_save_version() { done } -# client could mount several lustre +# client could mount several lustre +# +# XXX This function is kept for interoperability with old server (< 2.3.50), +# it should be removed whenever we drop the interoperability for such +# server. quota_type () { local fsname=${1:-$FSNAME} local rc=0 @@ -890,22 +1021,27 @@ quota_type () { return $rc } -restore_quota_type () { - local mntpt=${1:-$MOUNT} - local quota_type=$(quota_type $FSNAME | grep MDT | cut -d "=" -f2) - if [ ! "$old_QUOTA_TYPE" ] || [ "$quota_type" = "$old_QUOTA_TYPE" ]; then - return - fi - quota_save_version $old_QUOTA_TYPE +# XXX This function is kept for interoperability with old server (< 2.3.50), +# it should be removed whenever we drop the interoperability for such +# server. +restore_quota_old() { + local mntpt=${1:-$MOUNT} + local quota_type=$(quota_type $FSNAME | grep MDT | cut -d "=" -f2) + if [ ! "$old_QUOTA_TYPE" ] || + [ "$quota_type" = "$old_QUOTA_TYPE" ]; then + return + fi + quota_save_version $old_QUOTA_TYPE } -setup_quota(){ - local mntpt=$1 +# XXX This function is kept for interoperability with old server (< 2.3.50), +# it should be removed whenever we drop the interoperability for such +# server. +setup_quota_old(){ + local mntpt=$1 - if [ "$USE_OFD" = "yes" ]; then - $LFS quotacheck $mntpt || error "quotacheck failed" - return - fi + # no quota enforcement for now and accounting works out of the box + return # We need save the original quota_type params, and restore them after testing @@ -949,6 +1085,97 @@ setup_quota(){ done } +# get mdt quota type +mdt_quota_type() { + local varsvc=${SINGLEMDS}_svc + do_facet $SINGLEMDS $LCTL get_param -n \ + osd-$FSTYPE.${!varsvc}.quota_slave.enabled +} + +# get ost quota type +ost_quota_type() { + # All OSTs should have same quota type + local varsvc=ost1_svc + do_facet ost1 $LCTL get_param -n \ + osd-$FSTYPE.${!varsvc}.quota_slave.enabled +} + +# restore old quota type settings +restore_quota() { + if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then + restore_quota_old + return + fi + + if [ "$old_MDT_QUOTA_TYPE" ]; then + do_facet mgs $LCTL conf_param \ + $FSNAME.quota.mdt=$old_MDT_QUOTA_TYPE + fi + if [ "$old_OST_QUOTA_TYPE" ]; then + do_facet mgs $LCTL conf_param \ + $FSNAME.quota.ost=$old_OST_QUOTA_TYPE + fi +} + +setup_quota(){ + if [ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.3.50) ]; then + setup_quota_old $1 + return + fi + + # XXX remove it once all quota code landed + echo "skip quota setup" + return + + local mntpt=$1 + + # save old quota type & set new quota type + local mdt_qtype=$(mdt_quota_type) + local ost_qtype=$(ost_quota_type) + + echo "[HOST:$HOSTNAME] [old_mdt_qtype:$mdt_qtype]" \ + "[old_ost_qtype:$ost_qtype] [new_qtype:$QUOTA_TYPE]" + + export old_MDT_QUOTA_TYPE=$mdt_qtype + export old_OST_QUOTA_TYPE=$ost_qtype + + do_facet mgs $LCTL conf_param $FSNAME.quota.mdt=$QUOTA_TYPE || + error "set mdt quota type failed" + do_facet mgs $LCTL conf_param $FSNAME.quota.ost=$QUOTA_TYPE || + error "set ost quota type failed" + + local quota_usrs=$QUOTA_USERS + + # get_filesystem_size + local disksz=$(lfs df $mntpt | grep "filesystem summary:" | + awk '{print $3}') + local blk_soft=$((disksz + 1024)) + local blk_hard=$((blk_soft + blk_soft / 20)) # Go 5% over + + local inodes=$(lfs df -i $mntpt | grep "filesystem summary:" | + awk '{print $3}') + local i_soft=$inodes + local i_hard=$((i_soft + i_soft / 20)) + + echo "Total disk size: $disksz block-softlimit: $blk_soft" \ + "block-hardlimit: $blk_hard inode-softlimit: $i_soft" \ + "inode-hardlimit: $i_hard" + + local cmd + for usr in $quota_usrs; do + echo "Setting up quota on $HOSTNAME:$mntpt for $usr..." + for type in u g; do + cmd="$LFS setquota -$type $usr -b $blk_soft" + cmd="$cmd -B $blk_hard -i $i_soft -I $i_hard $mntpt" + echo "+ $cmd" + eval $cmd || error "$cmd FAILED!" + done + # display the quota status + echo "Quota settings for $usr : " + $LFS quota -v -u $usr $mntpt || true + done +} + zconf_mount() { local client=$1 local mnt=$2 @@ -1158,12 +1385,12 @@ facets_on_host () { echo $(comma_list $affected) } -facet_up () { - local facet=$1 - local host=${2:-$(facet_host $facet)} +facet_up() { + local facet=$1 + local host=${2:-$(facet_host $facet)} - local label=$(convert_facet2label $facet) - do_node $host lctl dl | awk '{print $4}' | grep -q $label + local label=$(convert_facet2label $facet) + do_node $host $LCTL dl | awk '{print $4}' | grep -q -x $label } facets_up_on_host () { @@ -1204,12 +1431,13 @@ remount_facet() { } reboot_facet() { - local facet=$1 - if [ "$FAILURE_MODE" = HARD ]; then - reboot_node $(facet_active_host $facet) - else - sleep 10 - fi + local facet=$1 + if [ "$FAILURE_MODE" = HARD ]; then + reboot_node $(facet_active_host $facet) + else + refresh_disk ${facet} + sleep 10 + fi } boot_node() { @@ -1503,28 +1731,63 @@ wait_update () { return 3 } -wait_update_facet () { - local facet=$1 - wait_update $(facet_active_host $facet) "$@" +wait_update_facet() { + local facet=$1 + shift + wait_update $(facet_active_host $facet) "$@" } -wait_delete_completed () { - local TOTALPREV=`lctl get_param -n osc.*.kbytesavail | \ - awk 'BEGIN{total=0}; {total+=$1}; END{print total}'` +sync_all_data() { + do_node $(osts_nodes) "lctl set_param -n osd*.*OS*.force_sync 1" 2>&1 | + grep -v 'Found no match' +} - local WAIT=0 - local MAX_WAIT=20 - while [ "$WAIT" -ne "$MAX_WAIT" ]; do - sleep 1 - TOTAL=`lctl get_param -n osc.*.kbytesavail | \ - awk 'BEGIN{total=0}; {total+=$1}; END{print total}'` - [ "$TOTAL" -eq "$TOTALPREV" ] && return 0 - echo "Waiting delete completed ... prev: $TOTALPREV current: $TOTAL " - TOTALPREV=$TOTAL - WAIT=$(( WAIT + 1)) - done - echo "Delete is not completed in $MAX_WAIT sec" - return 1 +wait_delete_completed_mds() { + local MAX_WAIT=${1:-20} + local mds2sync="" + local stime=`date +%s` + local etime + local node + local changes + + # find MDS with pending deletions + for node in $(mdts_nodes); do + changes=$(do_node $node "lctl get_param -n osc.*MDT*.sync_*" \ + 2>/dev/null | calc_sum) + if [ -z "$changes" ] || [ $changes -eq 0 ]; then + continue + fi + mds2sync="$mds2sync $node" + done + if [ "$mds2sync" == "" ]; then + return + fi + mds2sync=$(comma_list $mds2sync) + + # sync MDS transactions + do_nodes $mds2sync "lctl set_param -n osd*.*MD*.force_sync 1" + + # wait till all changes are sent and commmitted by OSTs + # for ldiskfs space is released upon execution, but DMU + # do this upon commit + + local WAIT=0 + while [ "$WAIT" -ne "$MAX_WAIT" ]; do + changes=$(do_nodes $mds2sync "lctl get_param -n osc.*MDT*.sync_*" \ + | calc_sum) + #echo "$node: $changes changes on all" + if [ "$changes" -eq "0" ]; then + etime=`date +%s` + #echo "delete took $((etime - stime)) seconds" + return + fi + sleep 1 + WAIT=$(( WAIT + 1)) + done + + etime=`date +%s` + echo "Delete is not completed in $((etime - stime)) seconds" + do_nodes $mds2sync "lctl get_param osc.*MDT*.sync_*" } wait_for_host() { @@ -1618,28 +1881,35 @@ wait_mds_ost_sync () { } wait_destroy_complete () { - echo "Waiting for destroy to be done..." - # MAX value shouldn't be big as this mean server responsiveness - # never increase this just to make test pass but investigate - # why it takes so long time - local MAX=5 - local WAIT=0 - while [ $WAIT -lt $MAX ]; do - local -a RPCs=($($LCTL get_param -n osc.*.destroys_in_flight)) - local con=1 - for ((i=0; i<${#RPCs[@]}; i++)); do - [ ${RPCs[$i]} -eq 0 ] && continue - # there are still some destroy RPCs in flight - con=0 - break; - done - sleep 1 - [ ${con} -eq 1 ] && return 0 # done waiting - echo "Waiting $WAIT secs for destroys to be done." - WAIT=$((WAIT + 1)) - done - echo "Destroys weren't done in $MAX sec." - return 1 + echo "Waiting for local destroys to complete" + # MAX value shouldn't be big as this mean server responsiveness + # never increase this just to make test pass but investigate + # why it takes so long time + local MAX=5 + local WAIT=0 + while [ $WAIT -lt $MAX ]; do + local -a RPCs=($($LCTL get_param -n osc.*.destroys_in_flight)) + local con=1 + local i + + for ((i=0; i<${#RPCs[@]}; i++)); do + [ ${RPCs[$i]} -eq 0 ] && continue + # there are still some destroy RPCs in flight + con=0 + break; + done + sleep 1 + [ ${con} -eq 1 ] && return 0 # done waiting + echo "Waiting ${WAIT}s for local destroys to complete" + WAIT=$((WAIT + 1)) + done + echo "Local destroys weren't done in $MAX sec." + return 1 +} + +wait_delete_completed() { + wait_delete_completed_mds $1 || return $? + wait_destroy_complete } wait_exit_ST () { @@ -1752,14 +2022,6 @@ facet_failover() { echo "Failing $facet on node $host" - # Make sure the client data is synced to disk. LU-924 - # - # We don't write client data synchrnously (to avoid flooding sync writes - # when there are many clients connecting), so if the server reboots before - # the client data reachs disk, the client data will be lost and the client - # will be evicted after recovery, which is not what we expected. - do_facet $facet "sync; sync; sync" - local affected=$(affected_facets $facet) shutdown_facet $facet @@ -1844,13 +2106,14 @@ fail_nodf() { } fail_abort() { - local facet=$1 - stop $facet - change_active $facet - wait_for_facet $facet - mount_facet $facet -o abort_recovery - clients_up || echo "first df failed: $?" - clients_up || error "post-failover df: $?" + local facet=$1 + stop $facet + refresh_disk ${facet} + change_active $facet + wait_for_facet $facet + mount_facet $facet -o abort_recovery + clients_up || echo "first df failed: $?" + clients_up || error "post-failover df: $?" } do_lmc() { @@ -2142,6 +2405,8 @@ single_local_node () { get_env_vars() { local var local value + local facets=$(get_facets) + local facet for var in ${!MODOPTS_*}; do value=${!var} @@ -2149,6 +2414,23 @@ get_env_vars() { done echo -n " USE_OFD=$USE_OFD" + + for facet in ${facets//,/ }; do + var=${facet}_FSTYPE + if [ -n "${!var}" ]; then + echo -n " $var=${!var}" + fi + done + + for var in MGSFSTYPE MDSFSTYPE OSTFSTYPE; do + if [ -n "${!var}" ]; then + echo -n " $var=${!var}" + fi + done + + if [ -n "$FSTYPE" ]; then + echo -n " FSTYPE=$FSTYPE" + fi } do_nodes() { @@ -2201,6 +2483,25 @@ do_facet() { do_node $HOST "$@" } +# Function: do_facet_random_file $FACET $FILE $SIZE +# Creates FILE with random content on the given FACET of given SIZE + +do_facet_random_file() { + local facet="$1" + local fpath="$2" + local fsize="$3" + local cmd="dd if=/dev/urandom of='$fpath' bs=$fsize count=1" + do_facet $facet "$cmd 2>/dev/null" +} + +do_facet_create_file() { + local facet="$1" + local fpath="$2" + local fsize="$3" + local cmd="dd if=/dev/zero of='$fpath' bs=$fsize count=1" + do_facet $facet "$cmd 2>/dev/null" +} + do_nodesv() { do_nodes --verbose "$@" } @@ -2751,26 +3052,66 @@ osc_ensure_active () { [ $period -lt $timeout ] || log "$count OST are inactive after $timeout seconds, give up" } +set_conf_param_and_check() { + local myfacet=$1 + local TEST=$2 + local PARAM=$3 + local ORIG=$(do_facet $myfacet "$TEST") + if [ $# -gt 3 ]; then + local FINAL=$4 + else + local -i FINAL + FINAL=$((ORIG + 5)) + fi + echo "Setting $PARAM from $ORIG to $FINAL" + do_facet mgs "$LCTL conf_param $PARAM='$FINAL'" || + error "conf_param $PARAM failed" + + wait_update $(facet_host $myfacet) "$TEST" "$FINAL" || + error "check $PARAM failed!" +} + init_param_vars () { - remote_mds_nodsh || - TIMEOUT=$(do_facet $SINGLEMDS "lctl get_param -n timeout") + remote_mds_nodsh || + TIMEOUT=$(do_facet $SINGLEMDS "lctl get_param -n timeout") - log "Using TIMEOUT=$TIMEOUT" + log "Using TIMEOUT=$TIMEOUT" - osc_ensure_active $SINGLEMDS $TIMEOUT - osc_ensure_active client $TIMEOUT + osc_ensure_active $SINGLEMDS $TIMEOUT + osc_ensure_active client $TIMEOUT - if [ $QUOTA_AUTO -ne 0 ]; then - if [ "$ENABLE_QUOTA" ]; then - echo "enable quota as required" - setup_quota $MOUNT || return 2 - else - echo "disable quota as required" - $LFS quotaoff -ug $MOUNT > /dev/null 2>&1 - fi - fi + local jobid_var + if [ -z "$(lctl get_param -n mdc.*.connect_flags | grep jobstats)" ]; then + jobid_var="none" + elif [ $JOBSTATS_AUTO -ne 0 ]; then + echo "enable jobstats, set job scheduler as $JOBID_VAR" + jobid_var=$JOBID_VAR + else + jobid_var=`$LCTL get_param -n jobid_var` + if [ $jobid_var != "disable" ]; then + echo "disable jobstats as required" + jobid_var="disable" + else + jobid_var="none" + fi + fi - return 0 + if [ $jobid_var == $JOBID_VAR -o $jobid_var == "disable" ]; then + do_facet mgs $LCTL conf_param $FSNAME.sys.jobid_var=$jobid_var + wait_update $HOSTNAME "$LCTL get_param -n jobid_var" \ + $jobid_var || return 1 + fi + + if [ $QUOTA_AUTO -ne 0 ]; then + if [ "$ENABLE_QUOTA" ]; then + echo "enable quota as required" + setup_quota $MOUNT || return 2 + else + echo "disable quota as required" + # $LFS quotaoff -ug $MOUNT > /dev/null 2>&1 + fi + fi + return 0 } nfs_client_mode () { @@ -2971,26 +3312,24 @@ cleanup_and_setup_lustre() { # Get all of the server target devices from a given server node and type. get_mnt_devs() { - local node=$1 - local type=$2 - local obd_type - local devs - local dev - - case $type in - mdt) obd_type="osd" ;; - ost) obd_type="obdfilter" ;; # needs to be fixed when OST also uses an OSD - *) echo "invalid server type" && return 1 ;; - esac + local node=$1 + local type=$2 + local devs + local dev - devs=$(do_node $node "lctl get_param -n $obd_type*.*.mntdev") - for dev in $devs; do - case $dev in - *loop*) do_node $node "losetup $dev" | \ - sed -e "s/.*(//" -e "s/).*//" ;; - *) echo $dev ;; - esac - done + if [ "$type" == ost ]; then + devs=$(get_obdfilter_param $node "" mntdev) + else + devs=$(do_node $node \ + "lctl get_param -n osd-*.$FSNAME-M*.mntdev") + fi + for dev in $devs; do + case $dev in + *loop*) do_node $node "losetup $dev" | \ + sed -e "s/.*(//" -e "s/).*//" ;; + *) echo $dev ;; + esac + done } # Get all of the server target devices. @@ -3040,6 +3379,9 @@ generate_db() { local ostidx local dev + [[ $(lustre_version_code $SINGLEMDS) -ne $(version_code 2.2.0) ]] || + { skip "Lustre 2.2.0 lacks the patch for LU-1255"; exit 0; } + check_shared_dir $SHARED_DIRECTORY || error "$SHARED_DIRECTORY isn't a shared directory" @@ -3083,11 +3425,11 @@ check_and_cleanup_lustre() { run_lfsck fi - if is_mounted $MOUNT; then - [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* || - error "remove sub-test dirs failed" - [ "$ENABLE_QUOTA" ] && restore_quota_type || true - fi + if is_mounted $MOUNT; then + [ -n "$DIR" ] && rm -rf $DIR/[Rdfs][0-9]* || + error "remove sub-test dirs failed" + [ "$ENABLE_QUOTA" ] && restore_quota || true + fi if [ "$I_UMOUNTED2" = "yes" ]; then restore_mount $MOUNT2 || error "restore $MOUNT2 failed" @@ -3260,6 +3602,10 @@ at_max_get() { at_get $1 at_max } +at_min_get() { + at_get $1 at_min +} + at_max_set() { local at_max=$1 shift @@ -3434,6 +3780,17 @@ stop_full_debug_logging() { debugrestore } +# prints bash call stack +log_trace_dump() { + echo " Trace dump:" + for (( i=1; i < ${#BASH_LINENO[*]} ; i++ )) ; do + local s=${BASH_SOURCE[$i]} + local l=${BASH_LINENO[$i-1]} + local f=${FUNCNAME[$i]} + echo " = $s:$l:$f()" + done +} + ################################## # Test interface ################################## @@ -3449,6 +3806,7 @@ error_noexit() { fi log " ${TESTSUITE} ${TESTNAME}: @@@@@@ ${TYPE}: $@ " + log_trace_dump mkdir -p $LOGDIR # We need to dump the logs on all nodes @@ -3462,11 +3820,11 @@ error_noexit() { } exit_status () { - local status=0 - local log=$TESTSUITELOG + local status=0 + local log=$TESTSUITELOG - [ -f "$log" ] && grep -q FAIL: $log && status=1 - exit $status + [ -f "$log" ] && grep -q FAIL $log && status=1 + exit $status } error() { @@ -3487,17 +3845,29 @@ error_ignore() { error_noexit "$@" } +error_and_remount() { + error_noexit "$@" + remount_client $MOUNT + exit 1 +} + skip_env () { $FAIL_ON_SKIP_ENV && error false $@ || skip $@ } -skip () { - echo - log " SKIP: ${TESTSUITE} ${TESTNAME} $@" - [ "$ALWAYS_SKIPPED" ] && \ - skip_logged ${TESTNAME} "$@" || true - [ "$TESTSUITELOG" ] && \ - echo "${TESTSUITE}: SKIP: $TESTNAME $@" >> $TESTSUITELOG || true +skip() { + echo + log " SKIP: $TESTSUITE $TESTNAME $@" + + if [[ -n "$ALWAYS_SKIPPED" ]]; then + skip_logged $TESTNAME "$@" + else + mkdir -p $LOGDIR + echo "$@" > $LOGDIR/skip + fi + + [[ -n "$TESTSUITELOG" ]] && + echo "$TESTSUITE: SKIP: $TESTNAME $@" >> $TESTSUITELOG || true } build_test_filter() { @@ -3565,41 +3935,36 @@ run_test() { echo -n "." return 0 fi + + LAST_SKIPPED="y" + ALWAYS_SKIPPED="y" testname=EXCEPT_$1 if [ ${!testname}x != x ]; then - LAST_SKIPPED="y" TESTNAME=test_$1 skip "skipping excluded test $1" return 0 fi testname=EXCEPT_$base if [ ${!testname}x != x ]; then - LAST_SKIPPED="y" TESTNAME=test_$1 skip "skipping excluded test $1 (base $base)" return 0 fi testname=EXCEPT_ALWAYS_$1 if [ ${!testname}x != x ]; then - LAST_SKIPPED="y" - ALWAYS_SKIPPED="y" TESTNAME=test_$1 skip "skipping ALWAYS excluded test $1" return 0 fi testname=EXCEPT_ALWAYS_$base if [ ${!testname}x != x ]; then - LAST_SKIPPED="y" - ALWAYS_SKIPPED="y" TESTNAME=test_$1 skip "skipping ALWAYS excluded test $1 (base $base)" return 0 fi testname=EXCEPT_SLOW_$1 if [ ${!testname}x != x ]; then - LAST_SKIPPED="y" TESTNAME=test_$1 skip "skipping SLOW test $1" return 0 fi testname=EXCEPT_SLOW_$base if [ ${!testname}x != x ]; then - LAST_SKIPPED="y" TESTNAME=test_$1 skip "skipping SLOW test $1 (base $base)" return 0 fi @@ -3647,13 +4012,15 @@ complete () { } pass() { - # Set TEST_STATUS here; will be used for logging the result - if [ -f $LOGDIR/err ]; then - TEST_STATUS="FAIL" - else - TEST_STATUS="PASS" - fi - echo "$TEST_STATUS $@" 2>&1 | tee -a $TESTSUITELOG + # Set TEST_STATUS here. It will be used for logging the result. + TEST_STATUS="PASS" + + if [[ -f $LOGDIR/err ]]; then + TEST_STATUS="FAIL" + elif [[ -f $LOGDIR/skip ]]; then + TEST_STATUS="SKIP" + fi + echo "$TEST_STATUS $@" 2>&1 | tee -a $TESTSUITELOG } check_mds() { @@ -3696,7 +4063,7 @@ banner() { run_one() { local testnum=$1 local message=$2 - tfile=f${testnum} + tfile=f.${TESTSUITE}.${testnum} export tdir=d0.${TESTSUITE}/d${base} export TESTNAME=test_$testnum local SAVE_UMASK=`umask` @@ -3722,42 +4089,49 @@ run_one() { # - test result is saved to data file # run_one_logged() { - local BEFORE=`date +%s` - local TEST_ERROR - local name=${TESTSUITE}.test_${1}.test_log.$(hostname -s).log - local test_log=$LOGDIR/$name - rm -rf $LOGDIR/err - local SAVE_UMASK=`umask` - umask 0022 - - echo - log_sub_test_begin test_${1} - (run_one $1 "$2") 2>&1 | tee -i $test_log - local RC=${PIPESTATUS[0]} - - [ $RC -ne 0 ] && [ ! -f $LOGDIR/err ] && \ - echo "test_$1 returned $RC" | tee $LOGDIR/err - - duration=$((`date +%s` - $BEFORE)) - pass "$1" "(${duration}s)" - [ -f $LOGDIR/err ] && TEST_ERROR=$(cat $LOGDIR/err) - log_sub_test_end $TEST_STATUS $duration "$RC" "$TEST_ERROR" + local BEFORE=`date +%s` + local TEST_ERROR + local name=${TESTSUITE}.test_${1}.test_log.$(hostname -s).log + local test_log=$LOGDIR/$name + rm -rf $LOGDIR/err + rm -rf $LOGDIR/skip + local SAVE_UMASK=`umask` + umask 0022 + + echo + log_sub_test_begin test_${1} + (run_one $1 "$2") 2>&1 | tee -i $test_log + local RC=${PIPESTATUS[0]} + + [ $RC -ne 0 ] && [ ! -f $LOGDIR/err ] && \ + echo "test_$1 returned $RC" | tee $LOGDIR/err + + duration=$((`date +%s` - $BEFORE)) + pass "$1" "(${duration}s)" + + if [[ -f $LOGDIR/err ]]; then + TEST_ERROR=$(cat $LOGDIR/err) + elif [[ -f $LOGDIR/skip ]]; then + TEST_ERROR=$(cat $LOGDIR/skip) + fi + log_sub_test_end $TEST_STATUS $duration "$RC" "$TEST_ERROR" - if [ -f $LOGDIR/err ]; then - $FAIL_ON_ERROR && exit $RC - fi + if [ -f $LOGDIR/err ]; then + $FAIL_ON_ERROR && exit $RC + fi - umask $SAVE_UMASK + umask $SAVE_UMASK - return $RC + return 0 } # # Print information of skipped tests to result.yml # skip_logged(){ - log_sub_test_begin $1 - log_sub_test_end "SKIP" "0" "0" "\"$2\"" + log_sub_test_begin $1 + shift + log_sub_test_end "SKIP" "0" "0" "$@" } canonical_path() { @@ -4055,6 +4429,63 @@ setstripe_nfsserver () { do_nodev $nfsserver lfs setstripe "$@" } +# Check and add a test group. +add_group() { + local group_id=$1 + local group_name=$2 + local rc=0 + + local gid=$(getent group $group_name | cut -d: -f3) + if [[ -n "$gid" ]]; then + [[ "$gid" -eq "$group_id" ]] || { + error_noexit "inconsistent group ID:" \ + "new: $group_id, old: $gid" + rc=1 + } + else + groupadd -g $group_id $group_name + rc=${PIPESTATUS[0]} + fi + + return $rc +} + +# Check and add a test user. +add_user() { + local user_id=$1 + shift + local user_name=$1 + shift + local group_name=$1 + shift + local home=$1 + shift + local opts="$@" + local rc=0 + + local uid=$(getent passwd $user_name | cut -d: -f3) + if [[ -n "$uid" ]]; then + if [[ "$uid" -eq "$user_id" ]]; then + local dir=$(getent passwd $user_name | cut -d: -f6) + if [[ "$dir" != "$home" ]]; then + mkdir -p $home + usermod -d $home $user_name + rc=${PIPESTATUS[0]} + fi + else + error_noexit "inconsistent user ID:" \ + "new: $user_id, old: $uid" + rc=1 + fi + else + mkdir -p $home + useradd -M -u $user_id -d $home -g $group_name $opts $user_name + rc=${PIPESTATUS[0]} + fi + + return $rc +} + check_runas_id_ret() { local myRC=0 local myRUNAS_UID=$1 @@ -4487,20 +4918,30 @@ wait_clients_import_state () { } oos_full() { - local -a AVAILA - local -a GRANTA - local OSCFULL=1 - AVAILA=($(do_nodes $(comma_list $(osts_nodes)) \ - $LCTL get_param obdfilter.*.kbytesavail)) - GRANTA=($(do_nodes $(comma_list $(osts_nodes)) \ - $LCTL get_param -n obdfilter.*.tot_granted)) - for ((i=0; i<${#AVAILA[@]}; i++)); do - local -a AVAIL1=(${AVAILA[$i]//=/ }) - GRANT=$((${GRANTA[$i]}/1024)) - echo -n $(echo ${AVAIL1[0]} | cut -d"." -f2) avl=${AVAIL1[1]} grnt=$GRANT diff=$((AVAIL1[1] - GRANT)) - [ $((AVAIL1[1] - GRANT)) -lt 400 ] && OSCFULL=0 && echo " FULL" || echo - done - return $OSCFULL + local -a AVAILA + local -a GRANTA + local -a TOTALA + local OSCFULL=1 + AVAILA=($(do_nodes $(comma_list $(osts_nodes)) \ + $LCTL get_param obdfilter.*.kbytesavail)) + GRANTA=($(do_nodes $(comma_list $(osts_nodes)) \ + $LCTL get_param -n obdfilter.*.tot_granted)) + TOTALA=($(do_nodes $(comma_list $(osts_nodes)) \ + $LCTL get_param -n obdfilter.*.kbytestotal)) + for ((i=0; i<${#AVAILA[@]}; i++)); do + local -a AVAIL1=(${AVAILA[$i]//=/ }) + local -a TOTAL=(${TOTALA[$i]//=/ }) + GRANT=$((${GRANTA[$i]}/1024)) + # allow 1% of total space in bavail because of delayed + # allocation with ZFS which might release some free space after + # txg commit. For small devices, we set a mininum of 8MB + local LIMIT=$((${TOTAL} / 100 + 8000)) + echo -n $(echo ${AVAIL1[0]} | cut -d"." -f2) avl=${AVAIL1[1]} \ + grnt=$GRANT diff=$((AVAIL1[1] - GRANT)) limit=${LIMIT} + [ $((AVAIL1[1] - GRANT)) -lt $LIMIT ] && OSCFULL=0 && \ + echo " FULL" || echo + done + return $OSCFULL } pool_list () { @@ -5227,3 +5668,101 @@ generate_string() { echo "$(head -c $size < /dev/zero | tr '\0' y)" } + +reformat_external_journal() { + if [ ! -z ${EJOURNAL} ]; then + local rcmd="do_facet ${SINGLEMDS}" + + echo "reformat external journal on ${SINGLEMDS}:${EJOURNAL}" + ${rcmd} mke2fs -O journal_dev ${EJOURNAL} || return 1 + fi +} + +# MDT file-level backup/restore +mds_backup_restore() { + local devname=$(mdsdevname ${SINGLEMDS//mds/}) + local mntpt=$(facet_mntpt brpt) + local rcmd="do_facet ${SINGLEMDS}" + local metaea=${TMP}/backup_restore.ea + local metadata=${TMP}/backup_restore.tgz + local opts=${MDS_MOUNT_OPTS} + + if ! ${rcmd} test -b ${devname}; then + opts=$(csa_add "$opts" -o loop) + fi + + echo "file-level backup/restore on ${SINGLEMDS}:${devname}" + + # step 1: build mount point + ${rcmd} mkdir -p $mntpt + # step 2: cleanup old backup + ${rcmd} rm -f $metaea $metadata + # step 3: mount dev + ${rcmd} mount -t ldiskfs $opts $devname $mntpt || return 1 + # step 4: backup metaea + echo "backup EA" + ${rcmd} "cd $mntpt && getfattr -R -d -m '.*' -P . > $metaea && cd -" || + return 2 + # step 5: backup metadata + echo "backup data" + ${rcmd} tar zcf $metadata -C $mntpt/ . > /dev/null 2>&1 || return 3 + # step 6: umount + ${rcmd} umount -d $mntpt || return 4 + # step 7: reformat external journal if needed + reformat_external_journal || return 5 + # step 8: reformat dev + echo "reformat new device" + add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS}) --backfstype ldiskfs \ + --reformat $devname > /dev/null || return 6 + # step 9: mount dev + ${rcmd} mount -t ldiskfs $opts $devname $mntpt || return 7 + # step 10: restore metadata + echo "restore data" + ${rcmd} tar zxfp $metadata -C $mntpt > /dev/null 2>&1 || return 8 + # step 11: restore metaea + echo "restore EA" + ${rcmd} "cd $mntpt && setfattr --restore=$metaea && cd - " || return 9 + # step 12: remove recovery logs + echo "remove recovery logs" + ${rcmd} rm -fv $mntpt/OBJECTS/* $mntpt/CATALOGS + # step 13: umount dev + ${rcmd} umount -d $mntpt || return 10 + # step 14: cleanup tmp backup + ${rcmd} rm -f $metaea $metadata +} + +# remove OI files +mds_remove_ois() { + local devname=$(mdsdevname ${SINGLEMDS//mds/}) + local mntpt=$(facet_mntpt brpt) + local rcmd="do_facet ${SINGLEMDS}" + local idx=$1 + local opts=${MDS_MOUNT_OPTS} + + if ! ${rcmd} test -b ${devname}; then + opts=$(csa_add "$opts" -o loop) + fi + + echo "remove OI files: idx=${idx}" + + # step 1: build mount point + ${rcmd} mkdir -p $mntpt + # step 2: mount dev + ${rcmd} mount -t ldiskfs $opts $devname $mntpt || return 1 + if [ -z $idx ]; then + # step 3: remove all OI files + ${rcmd} rm -fv $mntpt/oi.16* + elif [ $idx -lt 2 ]; then + ${rcmd} rm -fv $mntpt/oi.16.${idx} + else + local i + + # others, rm oi.16.[idx, idx * idx, idx ** ...] + for ((i=${idx}; i<64; i=$((i * idx)))); do + ${rcmd} rm -fv $mntpt/oi.16.${i} + done + fi + # step 4: umount + ${rcmd} umount -d $mntpt || return 2 + # OI files will be recreated when mounted as lustre next time. +}