[[ $MDS1_VERSION -ge $(version_code 2.10.56) ]] ||
skip "Need MDS version at least 2.10.56"
- local mdts=$(comma_list $(mdts_nodes))
+ local mdts=$(mdts_nodes)
# Create a user
changelog_register || error "first changelog_register failed"
# use all_char because created files should be evenly distributed
test_mkdir -c $MDSCOUNT -H all_char $DIR/$tdir ||
error "test_mkdir $tdir failed"
- log "$(date +%s): creating first files"
- for ((i = 0; i < MDSCOUNT * 2; i++)); do
- $LFS mkdir -i $((i%MDSCOUNT)) $DIR/$tdir/d$i.$((i/MDSCOUNT)) ||
- error "create $DIR/$tdir/d$i.$((i/MDSCOUNT)) failed"
+ log "$(date +%s): creating first dirs"
+ for ((idx = 0; idx < MDSCOUNT * 2; idx++)); do
+ local d=$DIR/$tdir/d$idx.$((idx/MDSCOUNT))
+ $LFS mkdir -i $((idx%MDSCOUNT)) $d ||
+ error "create $d on MDT$idx failed"
done
# check changelogs have been generated
local start=$SECONDS
local idle_time=$((MDSCOUNT * 5 + 5))
local nbcl=$(changelog_dump | wc -l)
- [[ $nbcl -eq 0 ]] && error "no changelogs found"
+ (( $nbcl != 0 )) || error "no changelogs found"
for param in "changelog_max_idle_time=$idle_time" \
"changelog_gc=1" \
#define OBD_FAIL_CAT_FREE_RECORDS 0x1313
do_nodes $mdts "$LCTL set_param fail_loc=0x1313 fail_val=3"
- for i in $(seq $MDSCOUNT); do
+ for ((i = 1; i <= $MDSCOUNT; i++)); do
cl_users=(${CL_USERS[mds$i]})
cl_user1[mds$i]="${cl_users[0]}"
cl_user2[mds$i]="${cl_users[1]}"
# Generate one more changelog to trigger GC at fail_loc for cl_user2.
# cl_user1 should be OK because it recently processed records.
- echo "$(date +%s): creating $((MDSCOUNT * 2)) files"
- for ((i = 0; i < MDSCOUNT * 2; i++)); do
- $LFS mkdir -i $((i%MDSCOUNT)) $DIR/$tdir/d$i.$((i/MDSCOUNT+2))||
- error "create $DIR/$tdir/d$i.$((i/MDSCOUNT+2)) failed"
+ echo "$(date +%s): creating $((MDSCOUNT * 2)) dirs"
+ for ((idx = 0; idx < MDSCOUNT * 2; idx++)); do
+ local d=$DIR/$tdir/d$idx.$((idx/MDSCOUNT+2))
+
+ $LFS mkdir -i $((idx%MDSCOUNT)) $d ||
+ error "create dir $d on MDT$idx failed"
done
# ensure gc thread is done
- for i in $(mdts_nodes); do
- wait_update $i "ps -e -o comm= | grep chlg_gc_thread" "" 20 ||
- error "$i: GC-thread not done"
+ for mds in ${mdts//,/ }; do
+ wait_update $mds "ps -e -o comm= | grep chlg_gc_thread" "" 20 ||
+ error "$mds: GC-thread not done"
done
local first_rec
[[ $MDS1_VERSION -ge $(version_code 2.14.55) ]] ||
skip "Need MDS version at least 2.14.55"
- local mdts=$(comma_list $(mdts_nodes))
+ local mdts=$(mdts_nodes)
# Create a user
changelog_register || error "first changelog_register failed"
# use all_char because created files should be evenly distributed
test_mkdir -c $MDSCOUNT -H all_char $DIR/$tdir ||
error "test_mkdir $tdir failed"
- for ((i = 0; i < MDSCOUNT; i++)); do
- $LFS mkdir -i $i $DIR/$tdir/d$i.1 $DIR/$tdir/d$i.2 ||
- error "create $DIR/$tdir/d$i.1 failed"
+ for ((idx = 0; idx < MDSCOUNT; idx++)); do
+ $LFS mkdir -i $idx $DIR/$tdir/d$idx.1 $DIR/$tdir/d$idx.2 ||
+ error "create $DIR/$tdir/d$idx.1 failed"
done
# check changelogs have been generated
done
local start=$SECONDS
- for i in $(seq $MDSCOUNT); do
+ for ((i=1; i <= $MDSCOUNT; i++)); do
cl_users=(${CL_USERS[mds$i]})
cl_user1[mds$i]="${cl_users[0]}"
cl_user2[mds$i]="${cl_users[1]}"
(( sleep2 > 0 )) && echo "sleep $sleep2 for interval" && sleep $sleep2
# Generate one more changelog to trigger GC at fail_loc for cl_user2.
# cl_user1 should be OK because it recently processed records.
- for ((i = 0; i < MDSCOUNT; i++)); do
- $LFS mkdir -i $i $DIR/$tdir/d$i.3 ||
- error "create $DIR/$tdir/d$i.3 failed"
+ for ((idx = 0; idx < MDSCOUNT; idx++)); do
+ $LFS mkdir -i $idx $DIR/$tdir/d$idx.3 ||
+ error "create $DIR/$tdir/d$idx.3 failed"
done
# ensure gc thread is done
- for i in $(mdts_nodes); do
- wait_update $i "ps -e -o comm= | grep chlg_gc_thread" "" 20 ||
- error "$i: GC-thread not done"
+ for mds in ${mdts//,/ }; do
+ wait_update $mds "ps -e -o comm= | grep chlg_gc_thread" "" 20 ||
+ error "$mds: GC-thread not done"
done
local first_rec
[[ $MDS1_VERSION -ge $(version_code 2.10.56) ]] ||
skip "Need MDS version at least 2.10.56"
- local mdts=$(comma_list $(mdts_nodes))
+ local mdts=$(mdts_nodes)
# Create a user
changelog_register || error "first changelog_register failed"
# use all_char because created files should be evenly distributed
test_mkdir -c $MDSCOUNT -H all_char $DIR/$tdir ||
error "test_mkdir $tdir failed"
- for ((i = 0; i < MDSCOUNT; i++)); do
- $LFS mkdir -i $i $DIR/$tdir/d$i.1 $DIR/$tdir/d$i.2 ||
- error "create $DIR/$tdir/d$i.1 failed"
+ for ((idx = 0; idx < MDSCOUNT; idx++)); do
+ $LFS mkdir -i $idx $DIR/$tdir/d$idx.1 $DIR/$tdir/d$idx.2 ||
+ error "create $DIR/$tdir/d$idx.1 failed"
done
# check changelogs have been generated
# force cl_user2 to be idle (1st part)
sleep 9
- for i in $(seq $MDSCOUNT); do
+ for ((i=1; i <= $MDSCOUNT; i++)); do
cl_users=(${CL_USERS[mds$i]})
cl_user1[mds$i]="${cl_users[0]}"
cl_user2[mds$i]="${cl_users[1]}"
# stop MDT to stop GC-thread, should be done in back-ground as it will
# block waiting for the thread to be released and exit
declare -A stop_pids
- for i in $(seq $MDSCOUNT); do
+ for ((i=1; i <= $MDSCOUNT; i++)); do
stop mds$i &
stop_pids[mds$i]=$!
done
- for i in $(mdts_nodes); do
+ for mds in ${mdts//,/ }; do
local facet
local nb=0
- local facets=$(facets_up_on_host $i)
+ local facets=$(facets_up_on_host $mds)
for facet in ${facets//,/ }; do
if [[ $facet == mds* ]]; then
done
# ensure each MDS's gc threads are still present and all in "R"
# state (OBD_FAIL_FORCE_GC_THREAD effect!)
- [[ $(do_node $i pgrep chlg_gc_thread | wc -l) -eq $nb ]] ||
- error "$i: expected $nb GC-thread"
- wait_update $i \
+ [[ $(do_node $mds pgrep chlg_gc_thread | wc -l) -eq $nb ]] ||
+ error "$mds: expected $nb GC-thread"
+ wait_update $mds \
"ps -C chlg_gc_thread -o state --no-headers | uniq" \
"R" 20 ||
- error "$i: GC-thread not found in R-state"
+ error "$mds: GC-thread not found in R-state"
# check umounts of each MDT on MDS have reached kthread_stop()
- [[ $(do_node $i pgrep umount | wc -l) -eq $nb ]] ||
- error "$i: expected $nb umount"
- wait_update $i \
+ [[ $(do_node $mds pgrep umount | wc -l) -eq $nb ]] ||
+ error "$mds: expected $nb umount"
+ wait_update $mds \
"ps -C umount -o state --no-headers | uniq" "D" 20 ||
- error "$i: umount not found in D-state"
+ error "$mds: umount not found in D-state"
done
# release all GC-threads
do_nodes $mdts $LCTL set_param fail_loc=0
# wait for MDT stop to complete
- for i in $(seq $MDSCOUNT); do
+ for ((i=1; i <= $MDSCOUNT; i++)); do
wait ${stop_pids[mds$i]} || error "mds$i: stop failed"
done
# via ldiskfs/zfs and llog_reader...
# re-start/mount MDTs
- for i in $(seq $MDSCOUNT); do
+ for ((i=1; i <= $MDSCOUNT; i++)); do
start mds$i $(mdsdevname $i) $MDS_MOUNT_OPTS ||
error "Fail to start mds$i"
done
local first_rec
- for i in $(seq $MDSCOUNT); do
+ for ((i=1; i <= $MDSCOUNT; i++)); do
# check cl_user1 still registered
changelog_users mds$i | grep -q "${cl_user1[mds$i]}" ||
error "mds$i: User ${cl_user1[mds$i]} not registered"
"during mount"
test_160i() {
-
- local mdts=$(comma_list $(mdts_nodes))
+ local mdts=$(mdts_nodes)
changelog_register || error "first changelog_register failed"
local i
local last_rec
declare -A LAST_REC
- for i in $(seq $MDSCOUNT); do
+ for ((i=1; i <= $MDSCOUNT; i++)); do
if changelog_users mds$i | grep "^cl"; then
# make sure new records are added with one user present
LAST_REC[mds$i]=$(changelog_users $SINGLEMDS |
createmany -m $DIR/$tdir/${tfile}bis $((MDSCOUNT * 2)) ||
error "create $DIR/$tdir/${tfile}bis failed"
- for i in $(seq $MDSCOUNT); do
+ for ((i=1; i <= $MDSCOUNT; i++)); do
last_rec=$(changelog_users $SINGLEMDS |
awk '/^current.index:/ { print $NF }')
echo "verify changelogs are on: $last_rec != ${LAST_REC[mds$i]}"
(( $MDS1_VERSION >= $(version_code 2.14.55) )) ||
skip "Need MDS version at least 2.14.55"
- local mdts=$(comma_list $(mdts_nodes))
+ local mdts=$(mdts_nodes)
#define OBD_FAIL_TIME_IN_CHLOG_USER 0x1314
do_nodes $mdts $LCTL set_param fail_loc=0x1314 \
# use all_char because created files should be evenly distributed
test_mkdir -c $MDSCOUNT -H all_char $DIR/$tdir ||
error "test_mkdir $tdir failed"
- for ((i = 0; i < MDSCOUNT; i++)); do
- $LFS mkdir -i $i $DIR/$tdir/d$i.1 $DIR/$tdir/d$i.2 ||
- error "create $DIR/$tdir/d$i.1 failed"
+ for ((idx = 0; idx < MDSCOUNT; idx++)); do
+ $LFS mkdir -i $idx $DIR/$tdir/d$idx.1 $DIR/$tdir/d$idx.2 ||
+ error "create $DIR/$tdir/d$idx.1 on MDT$idx failed"
done
# check changelogs have been generated
done
local start=$SECONDS
- for i in $(seq $MDSCOUNT); do
+ for ((i=1; i <= $MDSCOUNT; i++)); do
cl_users=(${CL_USERS[mds$i]})
cl_user1[mds$i]="${cl_users[0]}"
done
# ensure gc thread is done
- for node in $(mdts_nodes); do
- wait_update $node "pgrep chlg_gc_thread" "" 20 ||
- error "$node: GC-thread not done"
+ for mds in ${mdts//,/ }; do
+ wait_update $mds "pgrep chlg_gc_thread" "" 20 ||
+ error "$mds: GC-thread not done"
done
do_nodes $mdts $LCTL set_param fail_loc=0
return 0
fi
- nodes=$(comma_list $(mdts_nodes))
+ nodes=$(mdts_nodes)
echo "Starting gss daemon on mds: $nodes"
if $GSS_SK; then
# Start all versions, in case of switching
}
stop_gss_daemons() {
- local nodes=$(comma_list $(mdts_nodes))
+ local nodes=$(mdts_nodes)
send_sigint $nodes lsvcgssd lgssd
}
set_params_mdts() {
- local mdts=${1:-$(comma_list $(mdts_nodes))}
+ local mdts=${1:-$(mdts_nodes)}
shift || true
local params="${@:-$MDS_LCTL_SETPARAM_PARAM}"
}
sync_all_data_mdts() {
- do_nodes $(comma_list $(mdts_nodes)) \
- "lctl set_param -n os[cd]*.*MDT*.force_sync=1"
+ do_nodes $(mdts_nodes) "lctl set_param -n os[cd]*.*MDT*.force_sync=1"
}
sync_all_data_osts() {
local ost_name=$(ostname_from_index $ost_idx)
local facets=$(get_facets MDS)
- do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param -n \
+ do_nodes $(mdts_nodes) $LCTL set_param -n \
osp.*$ost_name*.reserved_mb_low=$lwm \
osp.*$ost_name*.reserved_mb_high=$hwm > /dev/null
local etime
local node
local changes
+ local mdts=$(mdts_nodes)
# find MDS with pending deletions
- for node in $(mdts_nodes); do
+ for node in ${mdts//,/ }; do
changes=$(do_node $node "$LCTL get_param -n osc.*MDT*.sync_*" \
2>/dev/null | calc_sum)
if [[ $changes -eq 0 ]]; then
local WAIT_TIMEOUT=${1:-$MAX}
local WAIT=0
local new_wait=true
- local list=$(comma_list $(mdts_nodes))
+ local list=$(mdts_nodes)
local cmd="$LCTL get_param -n osp.*osc*.old_sync_processed"
if ! do_facet $SINGLEMDS \
"$LCTL list_param osp.*osc*.old_sync_processed 2> /dev/null"
# why it takes so long time
local MAX=${1:-5}
local WAIT=0
- local list=$(comma_list $(mdts_nodes))
+ local mdts=$(mdts_nodes)
while [ $WAIT -lt $MAX ]; do
- local -a RPCs=($(do_nodes $list $LCTL get_param -n osp.*.destroys_in_flight))
+ local -a RPCs=($(do_nodes $mdts $LCTL get_param -n osp.*.destroys_in_flight))
local con=1
local i
[ -n "$delay" ] || error "fail to get maxage"
sleep $delay
- local nodes=$(comma_list $(mdts_nodes))
+ local mdts=$(mdts_nodes)
+
# initiate statfs RPC, all to all MDTs
- do_nodes $nodes $LCTL get_param -N osp.*MDT*MDT*.filesfree >&/dev/null
- do_nodes $nodes $LCTL get_param -N osp.*MDT*MDT*.filesfree >&/dev/null
+ do_nodes $mdts $LCTL get_param -N osp.*MDT*MDT*.filesfree >&/dev/null
+ do_nodes $mdts $LCTL get_param -N osp.*MDT*MDT*.filesfree >&/dev/null
}
client_up() {
awk '/^status/ { print \\\$2 }'" "completed" 60 ||
error "MDS${k} namespace isn't the expected 'completed'"
done
- local rep_mdt=$(do_nodes $(comma_list $(mdts_nodes)) \
- $LCTL get_param -n mdd.$FSNAME-*.lfsck_* |
- awk '/repaired/ { print $2 }' | calc_sum)
- local rep_ost=$(do_nodes $(osts_nodes) \
- $LCTL get_param -n obdfilter.$FSNAME-*.lfsck_* |
- awk '/repaired/ { print $2 }' | calc_sum)
- local repaired=$((rep_mdt + rep_ost))
- [ $repaired -eq 0 ] ||
+ local repaired=$(do_nodes $(tgts_nodes) \
+ "$LCTL get_param -n *.$FSNAME-*.lfsck_*" |
+ awk '/repaired/ { print $2 }' | calc_sum)
+ (( repaired == 0 )) ||
error "lfsck repaired $rep_mdt MDT and $rep_ost OST errors"
}
drop_mdt_ldlm_reply() {
#define OBD_FAIL_MDS_LDLM_REPLY_NET 0x157
RC=0
- local list=$(comma_list $(mdts_nodes))
+ local mdts=$(mdts_nodes)
- do_nodes $list lctl set_param fail_loc=0x157
+ do_nodes $mdts lctl set_param fail_loc=0x157
do_facet client "$@" || RC=$?
- do_nodes $list lctl set_param fail_loc=0
+ do_nodes $mdts lctl set_param fail_loc=0
return $RC
}
drop_mdt_ldlm_reply_once() {
#define OBD_FAIL_MDS_LDLM_REPLY_NET 0x157
RC=0
- local list=$(comma_list $(mdts_nodes))
+ local mdts=$(mdts_nodes)
- do_nodes $list lctl set_param fail_loc=0x80000157
+ do_nodes $mdts lctl set_param fail_loc=0x80000157
do_facet client "$@" || RC=$?
- do_nodes $list lctl set_param fail_loc=0
+ do_nodes $mdts lctl set_param fail_loc=0
return $RC
}
clear_failloc() {
- facet=$1
- pause=$2
+ local facet=$1
+ local pause=$2
+
sleep $pause
echo "clearing fail_loc on $facet"
do_facet $facet "lctl set_param fail_loc=0 2>/dev/null || true"
${!is_local}
}
-remote_node () {
+remote_node() {
local node=$1
! local_node $node
}
-remote_mds ()
+# return true if any MDT is on a remote node
+remote_mds()
{
+ local mdts=$(mdts_nodes)
local node
- for node in $(mdts_nodes); do
+
+ for node in ${mdts//,/ }; do
remote_node $node && return 0
done
+
return 1
}
+# return true if any MDT is on a remote node and no remote shell is configured
remote_mds_nodsh()
{
[ -n "$CLIENTONLY" ] && return 0 || true
# Get name of the active MGS node.
mgs_node () {
- echo -n $(facets_nodes $(get_facets MGS))
- }
+ echo -n $(facets_nodes $(get_facets MGS))
+}
# Get all of the active MDS nodes.
-mdts_nodes () {
- echo -n $(facets_nodes $(get_facets MDS))
+mdts_nodes() {
+ comma_list $(facets_nodes $(get_facets MDS))
}
# Get all of the active OSS nodes in a comma-separated list.
local host
local failover_host
local nodes
- local nodes_sort
local i
- for i in $(seq $MDSCOUNT); do
+ for ((i=1; i <= $MDSCOUNT; i++)); do
host=mds${i}_HOST
failover_host=mds${i}failover_HOST
nodes="$nodes ${!host} ${!failover_host}"
done
- [ -n "$nodes" ] || nodes="${mds_HOST} ${mdsfailover_HOST}"
- nodes_sort=$(for i in $nodes; do echo $i; done | sort -u)
- echo -n $nodes_sort
+ comma_list $nodes
}
# Get all of the OSS nodes, including active and passive nodes.
save_lustre_params $(get_facets MDS) \
mdt.*.enable_remote_dir_gid > $params_file
- do_nodes $(comma_list $(mdts_nodes)) \
- $LCTL set_param mdt.*.enable_remote_dir_gid=-1
+ do_nodes $(mdts_nodes) $LCTL set_param mdt.*.enable_remote_dir_gid=-1
stack_trap "restore_lustre_params < $params_file" EXIT
changelog_chmask() {
local mask=$1
- do_nodes $(comma_list $(mdts_nodes)) \
- $LCTL set_param mdd.*.changelog_mask="$mask"
+ do_nodes $(mdts_nodes) $LCTL set_param mdd.*.changelog_mask="$mask"
}
# usage: __changelog_clear FACET CL_USER [+]INDEX
git://git.whamcloud.com / fs / lustre-release.git / commitdiff