1. Add dev parameter to mkfs_opts, so it can check if it needs
--mgs for dev. combined_mgs_mdt is only useful for mdt0.
2. Add multi-node facet failover support, so it can fail multiple
targets at the same time.
3. Remove mdsn_HOST/failover_HOSTs in local.sh and add them to
facet_host.
4. replace MDSDEV1 and MGSDEV with mdsdevname and mgsdevname, because
these vars are not always defined.
Signed-off-by: wang di <di.wang@whamcloud.com>
Change-Id: I979b72375a731cf163c6811be3e705ce2298b52c
Reviewed-on: http://review.whamcloud.com/4520
Reviewed-by: Yu Jian <yujian@whamcloud.com>
Tested-by: Hudson
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
# facet hosts
mds_HOST=${mds_HOST:-`hostname`}
mdsfailover_HOST=${mdsfailover_HOST}
# facet hosts
mds_HOST=${mds_HOST:-`hostname`}
mdsfailover_HOST=${mdsfailover_HOST}
-mds1_HOST=${mds1_HOST:-$mds_HOST}
-mds1failover_HOST=${mds1failover_HOST:-$mdsfailover_HOST}
-mgs_HOST=${mgs_HOST:-$mds1_HOST}
+mgs_HOST=${mgs_HOST:-$mds_HOST}
ost_HOST=${ost_HOST:-`hostname`}
ostfailover_HOST=${ostfailover_HOST}
CLIENTS=""
ost_HOST=${ost_HOST:-`hostname`}
ostfailover_HOST=${ostfailover_HOST}
CLIENTS=""
DAEMONSIZE=${DAEMONSIZE:-500}
MDSCOUNT=${MDSCOUNT:-1}
DAEMONSIZE=${DAEMONSIZE:-500}
MDSCOUNT=${MDSCOUNT:-1}
-[ $MDSCOUNT -gt 4 ] && MDSCOUNT=4
-for num in $(seq $MDSCOUNT); do
- eval mds${num}_HOST=\$\{mds${num}_HOST:-$mds_HOST\}
- eval mds${num}failover_HOST=\$\{mds${num}failover_HOST:-$mdsfailover_HOST\}
-done
MDSDEVBASE=${MDSDEVBASE:-$TMP/${FSNAME}-mdt}
MDSSIZE=${MDSSIZE:-200000}
#
MDSDEVBASE=${MDSDEVBASE:-$TMP/${FSNAME}-mdt}
MDSSIZE=${MDSSIZE:-200000}
#
MDS_FS_MKFS_OPTS=${MDS_FS_MKFS_OPTS:-}
MDS_MOUNT_OPTS=${MDS_MOUNT_OPTS:-}
MDS_FS_MKFS_OPTS=${MDS_FS_MKFS_OPTS:-}
MDS_MOUNT_OPTS=${MDS_MOUNT_OPTS:-}
-MGSDEV=${MGSDEV:-$MDSDEV1}
MGSSIZE=${MGSSIZE:-$MDSSIZE}
MGSOPT=${MGSOPT:-}
MGS_FS_MKFS_OPTS=${MGS_FS_MKFS_OPTS:-}
MGSSIZE=${MGSSIZE:-$MDSSIZE}
MGSOPT=${MGSOPT:-}
MGS_FS_MKFS_OPTS=${MGS_FS_MKFS_OPTS:-}
# test 8-char fsname as well
local FSNAME2=test1234
# test 8-char fsname as well
local FSNAME2=test1234
- add fs2mds $(mkfs_opts mds1) --nomgs --mgsnode=$MGSNID \
+
+ add fs2mds $(mkfs_opts mds1 ${fs2mdsdev} ) --nomgs --mgsnode=$MGSNID \
--fsname=${FSNAME2} --reformat $fs2mdsdev $fs2mdsvdev || exit 10
--fsname=${FSNAME2} --reformat $fs2mdsdev $fs2mdsvdev || exit 10
- add fs2ost $(mkfs_opts ost1) --fsname=${FSNAME2} --reformat \
- $fs2ostdev $fs2ostvdev || exit 10
+ add fs2ost $(mkfs_opts ost1 ${fs2ostdev}) --fsname=${FSNAME2} \
+ --reformat $fs2ostdev $fs2ostvdev || exit 10
setup
start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && trap cleanup_24a EXIT INT
setup
start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && trap cleanup_24a EXIT INT
local fs2mdsdev=$(mdsdevname 1_2)
local fs2mdsvdev=$(mdsvdevname 1_2)
local fs2mdsdev=$(mdsdevname 1_2)
local fs2mdsvdev=$(mdsvdevname 1_2)
- add fs2mds $(mkfs_opts mds1) --mgs --fsname=${FSNAME}2 --reformat \
- $fs2mdsdev $fs2mdsvdev || exit 10
+ add fs2mds $(mkfs_opts mds1 ${fs2mdsdev} ) --mgs --fsname=${FSNAME}2 \
+ --reformat $fs2mdsdev $fs2mdsvdev || exit 10
setup
start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && return 2
cleanup || return 6
setup
start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && return 2
cleanup || return 6
mkfsoptions="--mkfsoptions=\\\"-J size=8\\\"" # See bug 17931.
fi
mkfsoptions="--mkfsoptions=\\\"-J size=8\\\"" # See bug 17931.
fi
- add fs2mds $(mkfs_opts mds1) --fsname=${FSNAME2} --reformat \
- $mkfsoptions $fs2mdsdev $fs2mdsvdev || exit 10
- add fs2ost $(mkfs_opts ost1) --mgsnode=$MGSNID --fsname=${FSNAME2} \
- --index=8191 --reformat $fs2ostdev $fs2ostvdev || exit 10
+ add fs2mds $(mkfs_opts mds1 ${fs2mdsdev}) --mgs --fsname=${FSNAME2} \
+ --reformat $mkfsoptions $fs2mdsdev $fs2mdsvdev || exit 10
+ add fs2ost $(mkfs_opts ost1 ${fs2ostdev}) --mgsnode=$MGSNID \
+ --fsname=${FSNAME2} --index=8191 --reformat $fs2ostdev \
+ $fs2ostvdev || exit 10
start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && trap cleanup_24a EXIT INT
start fs2ost $fs2ostdev $OST_MOUNT_OPTS
start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS && trap cleanup_24a EXIT INT
start fs2ost $fs2ostdev $OST_MOUNT_OPTS
local fs2ostvdev=$(ostvdevname 1_2)
local fs3ostvdev=$(ostvdevname 2_2)
local fs2ostvdev=$(ostvdevname 1_2)
local fs3ostvdev=$(ostvdevname 2_2)
- add fs2mds $(mkfs_opts mds1) --fsname=${FSNAME2} --reformat \
- $fs2mdsdev $fs2mdsvdev || exit 10
+ add fs2mds $(mkfs_opts mds1 ${fs2mdsdev}) --mgs --fsname=${FSNAME2} \
+ --reformat $fs2mdsdev $fs2mdsvdev || exit 10
# XXX after we support non 4K disk blocksize in ldiskfs, specify a
# different one than the default value here.
# XXX after we support non 4K disk blocksize in ldiskfs, specify a
# different one than the default value here.
- add fs2ost $(mkfs_opts ost1) --mgsnode=$MGSNID --fsname=${FSNAME2} \
- --reformat $fs2ostdev $fs2ostvdev || exit 10
- add fs3ost $(mkfs_opts ost1) --mgsnode=$MGSNID --fsname=${FSNAME2} \
- --reformat $fs3ostdev $fs3ostvdev || exit 10
+ add fs2ost $(mkfs_opts ost1 ${fs2ostdev}) --mgsnode=$MGSNID \
+ --fsname=${FSNAME2} --reformat $fs2ostdev $fs2ostvdev || exit 10
+ add fs3ost $(mkfs_opts ost1 ${fs3ostdev}) --mgsnode=$MGSNID \
+ --fsname=${FSNAME2} --reformat $fs3ostdev $fs3ostvdev || exit 10
start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS
start fs2ost $fs2ostdev $OST_MOUNT_OPTS
start fs2mds $fs2mdsdev $MDS_MOUNT_OPTS
start fs2ost $fs2ostdev $OST_MOUNT_OPTS
- add mds1 $(mkfs_opts mds1) --reformat $mdsdev $mdsvdev ||
- exit 10
- add ost1 $(mkfs_opts ost1) --index=$i --reformat \
- $(ostdevname 1) $(ostvdevname 1)
+ add mds1 $(mkfs_opts mds1 ${mdsdev}) --reformat $mdsdev \
+ $mdsvdev || exit 10
+ add ost1 $(mkfs_opts ost1 $(ostdevname 1)) --index=$i \
+ --reformat $(ostdevname 1) $(ostvdevname 1)
setup_noconfig
stopall
setup_noconfig
setup_noconfig
stopall
setup_noconfig
local mds_journal_size_orig=$MDSJOURNALSIZE
MDSJOURNALSIZE=16
local mds_journal_size_orig=$MDSJOURNALSIZE
MDSJOURNALSIZE=16
- add mds1 $(mkfs_opts mds1) --reformat $(mdsdevname 1) $(mdsvdevname 1)
- add ost1 $(mkfs_opts ost1) --index=1000 --reformat \
+
+ for num in $(seq 1 $MDSCOUNT); do
+ add mds${num} $(mkfs_opts mds${num} $(mdsdevname $num)) \
+ --reformat $(mdsdevname $num) $(mdsvdevname $num)
+ done
+ add ost1 $(mkfs_opts ost1 $(ostdevname 1)) --index=1000 --reformat \
$(ostdevname 1) $(ostvdevname 1)
$(ostdevname 1) $(ostvdevname 1)
- add ost2 $(mkfs_opts ost2) --index=10000 --reformat \
+ add ost2 $(mkfs_opts ost2 $(ostdevname 2)) --index=10000 --reformat \
$(ostdevname 2) $(ostvdevname 2)
start_mgsmds
$(ostdevname 2) $(ostvdevname 2)
start_mgsmds
run_test 59 "writeconf mount option"
test_60() { # LU-471
run_test 59 "writeconf mount option"
test_60() { # LU-471
if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then
skip "Only applicable to ldiskfs-based MDTs"
return
fi
if [ $(facet_fstype $SINGLEMDS) != ldiskfs ]; then
skip "Only applicable to ldiskfs-based MDTs"
return
fi
- add mds1 $(mkfs_opts mds1) \
- --mkfsoptions='\" -E stride=64 -O ^uninit_bg\"' --reformat \
- $(mdsdevname 1) $(mdsvdevname 1) || exit 10
+ for num in $(seq $MDSCOUNT); do
+ add mds${num} $(mkfs_opts mds${num} $(mdsdevname $num)) \
+ --mkfsoptions='\" -E stride=64 -O ^uninit_bg\"' \
+ --reformat $(mdsdevname $num) $(mdsvdevname $num) ||
+ exit 10
+ done
dump=$(do_facet $SINGLEMDS dumpe2fs $(mdsdevname 1))
rc=${PIPESTATUS[0]}
dump=$(do_facet $SINGLEMDS dumpe2fs $(mdsdevname 1))
rc=${PIPESTATUS[0]}
if [ $(facet_fstype $SINGLEMDS) == ldiskfs ] &&
! large_xattr_enabled; then
reformat=true
if [ $(facet_fstype $SINGLEMDS) == ldiskfs ] &&
! large_xattr_enabled; then
reformat=true
- local mds_dev=$(mdsdevname ${SINGLEMDS//mds/})
LDISKFS_MKFS_OPTS+=" -O large_xattr"
LDISKFS_MKFS_OPTS+=" -O large_xattr"
- add $SINGLEMDS $(mkfs_opts $SINGLEMDS) --reformat $mds_dev ||
- error "reformatting $mds_dev failed"
+
+ for num in $(seq $MDSCOUNT); do
+ add mds${num} $(mkfs_opts mds$num $(mdsdevname $num)) \
+ --reformat $(mdsdevname $num) $(mdsvdevname $num) ||
+ error "add mds $num failed"
+ done
fi
setup_noconfig || error "setting up the filesystem failed"
fi
setup_noconfig || error "setting up the filesystem failed"
- local facet=$1
- local sleep_time=$2
- local host=$(facet_active_host $facet)
-
- echo "Failing $facet on node $host"
-
- local affected=$(affected_facets $facet)
+ local facets=$1
+ local sleep_time=$2
+ local -a affecteds
+ local facet
+ local total=0
+ local index=0
+ local skip
+
+ #Because it will only get up facets, we need get affected
+ #facets before shutdown
+ #For HARD Failure mode, it needs make sure facets on the same
+ #HOST will only be shutdown and reboot once
+ for facet in ${facets//,/ }; do
+ local affected_facet
+ skip=0
+ #check whether facet has been included in other affected facets
+ for ((index=0; index<$total; index++)); do
+ [[ *,$facet,* == ,${affecteds[index]}, ]] && skip=1
+ done
+ if [ $skip -eq 0 ]; then
+ affecteds[$total]=$(affected_facets $facet)
+ total=$((total+1))
+ fi
+ done
- echo affected facets: $affected
+ for ((index=0; index<$total; index++)); do
+ facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1)
+ local host=$(facet_active_host $facet)
+ echo "Failing ${affecteds[index]} on $host"
+ shutdown_facet $facet
+ done
- [ -n "$sleep_time" ] && sleep $sleep_time
+ for ((index=0; index<$total; index++)); do
+ facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1)
+ echo reboot facets: ${affecteds[index]}
- change_active $affected
+ change_active ${affecteds[index]}
- wait_for_facet $affected
- # start mgs first if it is affected
- if ! combined_mgs_mds && list_member $affected mgs; then
- mount_facet mgs || error "Restart of mgs failed"
- fi
- # FIXME; has to be changed to mount all facets concurrently
- affected=$(exclude_items_from_list $affected mgs)
- mount_facets $affected
+ wait_for_facet ${affecteds[index]}
+ # start mgs first if it is affected
+ if ! combined_mgs_mds &&
+ list_member ${affecteds[index]} mgs; then
+ mount_facet mgs || error "Restart of mgs failed"
+ fi
+ # FIXME; has to be changed to mount all facets concurrently
+ affected=$(exclude_items_from_list ${affecteds[index]} mgs)
+ echo mount facets: ${affecteds[index]}
+ mount_facets ${affecteds[index]}
+ done
+ local facet=$1
+ local varname
- [ "$facet" == client ] && echo -n $HOSTNAME && return
- varname=${facet}_HOST
- if [ -z "${!varname}" ]; then
- if [ "${facet:0:3}" == "ost" ]; then
- eval ${facet}_HOST=${ost_HOST}
- fi
- fi
- echo -n ${!varname}
+ [ "$facet" == client ] && echo -n $HOSTNAME && return
+ varname=${facet}_HOST
+ if [ -z "${!varname}" ]; then
+ if [ "${facet:0:3}" == "ost" ]; then
+ eval export ${facet}_HOST=${ost_HOST}
+ elif [ "${facet:0:3}" == "mdt" -o \
+ "${facet:0:3}" == "mds" -o \
+ "${facet:0:3}" == "mgs" ]; then
+ eval export ${facet}_HOST=${mds_HOST}
+ fi
+ fi
+ echo -n ${!varname}
}
facet_failover_host() {
local facet=$1
}
facet_failover_host() {
local facet=$1
var=${facet}failover_HOST
if [ -n "${!var}" ]; then
var=${facet}failover_HOST
if [ -n "${!var}" ]; then
+ if [ "${facet:0:3}" == "mdt" -o "${facet:0:3}" == "mds" -o \
+ "${facet:0:3}" == "mgs" ]; then
+
+ eval export ${facet}failover_host=${mds_HOST}
+ echo ${mds_HOST}
+ return
+ fi
+
if [[ $facet == ost* ]]; then
if [[ $facet == ost* ]]; then
- var=ostfailover_HOST
- if [ -n "${!var}" ]; then
- echo ${!var}
- return
- fi
+ eval export ${facet}failover_host=${ost_HOST}
+ echo ${ost_HOST}
+ return
+ local DEVNAME=MGSDEV
+ local MDSDEV1=$(mdsdevname 1)
- local fstype=$(facet_fstype mds$num)
+ local fstype=$(facet_fstype mds1)
case $fstype in
ldiskfs )
case $fstype in
ldiskfs )
mgsvdevname() {
DEVNAME=MGSDEV
mgsvdevname() {
DEVNAME=MGSDEV
- local fstype=$(facet_fstype mds$num)
+ local fstype=$(facet_fstype mds1)
case $fstype in
ldiskfs )
case $fstype in
ldiskfs )
- [[ $MDSDEV1 = $MGSDEV ]] && [[ $mds1_HOST = $mgs_HOST ]]
+ [[ "$(mdsdevname 1)" = "$(mgsdevname)" ]] &&
+ [[ "$(facet_host mds1)" = "$(facet_host mgs)" ]]
mkfs_opts() {
local facet=$1
mkfs_opts() {
local facet=$1
local type=$(facet_type $facet)
local index=$(($(facet_number $facet) - 1))
local fstype=$(facet_fstype $facet)
local type=$(facet_type $facet)
local index=$(($(facet_number $facet) - 1))
local fstype=$(facet_fstype $facet)
+ local host=$(facet_host $facet)
local opts
local fs_mkfs_opts
local var
local opts
local fs_mkfs_opts
local var
- if [ $type == MGS ] || ( [ $type == MDS ] && combined_mgs_mds ); then
+ if [ $type == MGS ] || ( [ $type == MDS ] &&
+ [ "$dev" == $(mgsdevname) ] &&
+ [ "$host" == "$(facet_host mgs)" ] ); then
opts="--mgs"
else
opts="--mgsnode=$MGSNID"
opts="--mgs"
else
opts="--mgsnode=$MGSNID"
echo Formatting mgs, mds, osts
if ! combined_mgs_mds ; then
echo "Format mgs: $(mgsdevname)"
echo Formatting mgs, mds, osts
if ! combined_mgs_mds ; then
echo "Format mgs: $(mgsdevname)"
- add mgs $(mkfs_opts mgs) --reformat $(mgsdevname) \
- $(mgsvdevname) ${quiet:+>/dev/null} || exit 10
- fi
+ add mgs $(mkfs_opts mgs $(mgsdevname)) --reformat \
+ $(mgsdevname) $(mgsvdevname) ${quiet:+>/dev/null} || exit 10
+ fi
- for num in `seq $MDSCOUNT`; do
- echo "Format mds$num: $(mdsdevname $num)"
- add mds$num $(mkfs_opts mds$num) --reformat \
- $(mdsdevname $num) $(mdsvdevname $num) \
- ${quiet:+>/dev/null} || exit 10
- done
+ for num in $(seq $MDSCOUNT); do
+ echo "Format mds$num: $(mdsdevname $num)"
+ add mds$num $(mkfs_opts mds$num $(mdsdevname ${num})) \
+ --reformat $(mdsdevname $num) $(mdsvdevname $num) \
+ ${quiet:+>/dev/null} || exit 10
+ done
- for num in `seq $OSTCOUNT`; do
- echo "Format ost$num: $(ostdevname $num)"
- add ost$num $(mkfs_opts ost$num) --reformat \
- $(ostdevname $num) $(ostvdevname ${num}) \
- ${quiet:+>/dev/null} || exit 10
- done
+ for num in $(seq $OSTCOUNT); do
+ echo "Format ost$num: $(ostdevname $num)"
+ add ost$num $(mkfs_opts ost$num $(ostdevname ${num})) \
+ --reformat $(ostdevname $num) $(ostvdevname ${num}) \
+ ${quiet:+>/dev/null} || exit 10
+ done
local varname=${facet}failover_HOST
if [ -z "${!varname}" ]; then
local varname=${facet}failover_HOST
if [ -z "${!varname}" ]; then
- eval $varname=$(facet_host $facet)
+ eval export $varname=$(facet_host $facet)
+ varname=${facet}_HOST
+ if [ -z "${!varname}" ]; then
+ eval export $varname=$(facet_host $facet)
+ fi
+
# ${facet}failover_dev is set in cfg file
varname=${facet}failover_dev
if [ -n "${!varname}" ] ; then
# ${facet}failover_dev is set in cfg file
varname=${facet}failover_dev
if [ -n "${!varname}" ] ; then
- if ! remote_mds_nodsh; then
- for num in `seq $MDSCOUNT`; do
- DEVNAME=`mdsdevname $num`
- init_facet_vars mds$num $DEVNAME $MDS_MOUNT_OPTS
- done
- fi
+ if ! remote_mds_nodsh; then
+ for num in $(seq $MDSCOUNT); do
+ DEVNAME=`mdsdevname $num`
+ eval export MDSDEV${num}=$DEVNAME
+ init_facet_vars mds$num $DEVNAME $MDS_MOUNT_OPTS
+ done
+ fi
+ eval export MGSDEV=$(mgsdevname)
combined_mgs_mds || init_facet_vars mgs $(mgsdevname) $MGS_MOUNT_OPTS
combined_mgs_mds || init_facet_vars mgs $(mgsdevname) $MGS_MOUNT_OPTS
- remote_ost_nodsh && return
-
- for num in `seq $OSTCOUNT`; do
- DEVNAME=`ostdevname $num`
- init_facet_vars ost$num $DEVNAME $OST_MOUNT_OPTS
- done
+ if ! remote_ost_nodsh; then
+ for num in $(seq $OSTCOUNT); do
+ DEVNAME=$(ostdevname $num)
+ eval export OSTDEV${num}=$DEVNAME
+ init_facet_vars ost$num $DEVNAME $OST_MOUNT_OPTS
+ done
+ fi