# output: No return values, environment variables are exported
get_lustre_env() {
+ if ! $RPC_MODE; then
+ export mds1_FSTYPE=${mds1_FSTYPE:-$(facet_fstype mds1)}
+ export ost1_FSTYPE=${ost1_FSTYPE:-$(facet_fstype ost1)}
- export mds1_FSTYPE=${mds1_FSTYPE:-$(facet_fstype mds1)}
- export ost1_FSTYPE=${ost1_FSTYPE:-$(facet_fstype ost1)}
-
- export MGS_VERSION=$(lustre_version_code mgs)
- export MDS1_VERSION=$(lustre_version_code mds1)
- export OST1_VERSION=$(lustre_version_code ost1)
- export CLIENT_VERSION=$(lustre_version_code client)
+ export MGS_VERSION=$(lustre_version_code mgs)
+ export MDS1_VERSION=$(lustre_version_code mds1)
+ export OST1_VERSION=$(lustre_version_code ost1)
+ export CLIENT_VERSION=$(lustre_version_code client)
+ fi
# Prefer using "mds1" directly instead of SINGLEMDS.
# Keep this for compat until it is removed from scripts.
# code is useful for comparison two version strings to see which is newer.
version_code() {
# split arguments like "1.8.6-wc3" into "1", "8", "6", "3"
- eval set -- $(tr "[:punct:][a-z]" " " <<< $*)
+ eval set -- $(tr "[:punct:][a-zA-Z]" " " <<< $*)
echo -n $(((${1:-0}<<24) | (${2:-0}<<16) | (${3:-0}<<8) | (${4:-0})))
}
mount_facets () {
local facets=${1:-$(get_facets)}
local facet
+ local -a mountpids
+ local total=0
+ local ret=0
for facet in ${facets//,/ }; do
- mount_facet $facet
+ mount_facet $facet &
+ mountpids[total]=$!
+ total=$((total+1))
+ done
+ for ((index=0; index<$total; index++)); do
+ wait ${mountpids[index]}
local RC=$?
[ $RC -eq 0 ] && continue
else
error "Restart of $facet failed!"
fi
- return $RC
+ ret=$RC
done
+ return $ret
}
#
skip=0
#check whether facet has been included in other affected facets
for ((index=0; index<$total; index++)); do
- [[ *,$facet,* == ,${affecteds[index]}, ]] && skip=1
+ [[ ,${affecteds[index]}, == *,$facet,* ]] && skip=1
done
if [ $skip -eq 0 ]; then
shutdown_facet $facet
done
- $E2FSCK_ON_MDT0 && (run_e2fsck $(facet_active_host $SINGLEMDS) \
- $(mdsdevname 1) "-n" || error "Running e2fsck")
+ echo "$(date +'%H:%M:%S (%s)') shut down"
- for ((index=0; index<$total; index++)); do
- facet=$(echo ${affecteds[index]} | tr -s " " | cut -d"," -f 1)
- echo reboot facets: ${affecteds[index]}
+ local hostlist
+ local waithostlist
- reboot_facet $facet $sleep_time
+ for facet in ${facets//,/ }; do
+ local host=$(facet_active_host $facet)
- change_active ${affecteds[index]}
+ hostlist=$(expand_list $hostlist $host)
+ if [ $(facet_host $facet) = \
+ $(facet_failover_host $facet) ]; then
+ waithostlist=$(expand_list $waithostlist $host)
+ fi
+ done
- wait_for_facet ${affecteds[index]}
+ if [ "$FAILURE_MODE" = HARD ]; then
+ for host in ${hostlist//,/ }; do
+ reboot_node $host
+ done
+ echo "$(date +'%H:%M:%S (%s)') $hostlist rebooted"
+ # We need to wait the rebooted hosts in case if
+ # facet_HOST == facetfailover_HOST
+ if ! [ -z "$waithostlist" ]; then
+ wait_for_host $waithostlist
+ if $LOAD_MODULES_REMOTE; then
+ echo "loading modules on $waithostlist"
+ do_rpc_nodes $waithostlist load_modules_local
+ fi
+ fi
+ else
+ sleep 10
+ fi
+
+ if [[ " ${affecteds[@]} " =~ " $SINGLEMDS " ]]; then
+ change_active $SINGLEMDS
+ fi
+
+ $E2FSCK_ON_MDT0 && (run_e2fsck $(facet_active_host $SINGLEMDS) \
+ $(facet_device $SINGLEMDS) "-n" || error "Running e2fsck")
+
+ local -a mountpids
+
+ for ((index=0; index<$total; index++)); do
+ if [[ ${affecteds[index]} != $SINGLEMDS ]]; then
+ change_active ${affecteds[index]}
+ fi
if $GSS_SK; then
init_gss
init_facets_vars_simple
affecteds[index]=$(exclude_items_from_list \
${affecteds[index]} mgs)
fi
- # FIXME; has to be changed to mount all facets concurrently
if [ -n "${affecteds[index]}" ]; then
echo mount facets: ${affecteds[index]}
- mount_facets ${affecteds[index]}
+ mount_facets ${affecteds[index]} &
+ mountpids[index]=$!
+ fi
+ done
+ for ((index=0; index<$total; index++)); do
+ if [ -n "${affecteds[index]}" ]; then
+ wait ${mountpids[index]}
fi
+
if $GSS_SK; then
do_nodes $(comma_list $(all_nodes)) \
"keyctl show | grep lustre | cut -c1-11 |
xargs -IX keyctl setperm X 0x3f3f3f3f"
fi
done
+ echo "$(date +'%H:%M:%S (%s)') targets are mounted"
+
+ if [ "$FAILURE_MODE" = HARD ]; then
+ hostlist=$(exclude_items_from_list $hostlist $waithostlist)
+ if ! [ -z "$hostlist" ]; then
+ wait_for_host $hostlist
+ if $LOAD_MODULES_REMOTE; then
+ echo "loading modules on $hostlist"
+ do_rpc_nodes $hostlist load_modules_local
+ fi
+ fi
+ fi
+
+ echo "$(date +'%H:%M:%S (%s)') facet_failover done"
}
replay_barrier() {
fi
if combined_mgs_mds && [ $facet == "mgs" ] &&
- [ -z $mds1failover_HOST ]; then
- temp=mds1failover_HOST
- echo ${!temp}
+ [ -n "$mds1failover_HOST" ]; then
+ echo $mds1failover_HOST
return
fi
return ${PIPESTATUS[0]}
}
+##
+# Execute exact command line on host
+#
+# The \a host may be on a local or remote node, which is determined at
+# the time the command is run. Does careful argument quotation to
+# ensure that the exact command line is executed without any globbing,
+# substitution, or shell interpretation on the remote side. Does not
+# support --verbose or --quiet. Does not include "$host: " prefixes on
+# output. See also do_facet_vp().
+#
+# usage: do_node_vp "$host" "$command" "$arg"...
+do_node_vp() {
+ local host="$1"
+ shift
+
+ if [[ "$host" == "$HOSTNAME" ]]; then
+ sh -c "$(printf -- ' %q' "$@")"
+ return $?
+ fi
+
+ if [[ "${PDSH}" != *pdsh* || "${PDSH}" != *-S* ]]; then
+ echo "cannot run '$*' on host '${host}' with PDSH='${PDSH}'" >&2
+ return 128
+ fi
+
+ # -N Disable hostname: prefix on lines of output.
+
+ $PDSH "${host}" -N "cd $RPWD; PATH=\$PATH:$RLUSTRE/utils:$RLUSTRE/tests:/sbin:/usr/sbin; export LUSTRE=$RLUSTRE; $(printf -- ' %q' "$@")"
+}
+
single_local_node () {
[ "$1" = "$HOSTNAME" ]
}
do_node $verbose $quiet $host "$@"
}
+##
+# Execute exact command line on the host of a facet
+#
+# The \a facet (service) may be on a local or remote node, which is
+# determined at the time the command is run. Does careful argument
+# quotation to ensure that the exact command line is executed without
+# any globbing, substitution, or shell interpretation on the remote
+# side. Does not support --verbose or --quiet. Does not include
+# "$host: " prefixes on output.
+#
+# usage: do_facet_vp "$facet" "$command" "$arg"...
+do_facet_vp() {
+ local facet="$1"
+ local host=$(facet_active_host "$facet")
+ shift
+
+ if [[ -z "$host" ]]; then
+ echo "no host defined for facet ${facet}" >&2
+ exit 1
+ fi
+
+ do_node_vp "$host" "$@"
+}
+
# Function: do_facet_random_file $FACET $FILE $SIZE
# Creates FILE with random content on the given FACET of given SIZE
# The add fn does rm ${facet}active file, this would be enough
# if we use do_facet <facet> only after the facet added, but
# currently we use do_facet mds in local.sh
+ local num
for num in `seq $MDSCOUNT`; do
stop mds$num -f
rm -f ${TMP}/mds${num}active
local MSG="$*"
# Get rid of '
MSG=${MSG//\'/\\\'}
+ MSG=${MSG//\*/\\\*}
MSG=${MSG//\(/\\\(}
MSG=${MSG//\)/\\\)}
MSG=${MSG//\;/\\\;}
# Sanity check: exclude the dup entries
RCLIENTS=$(for i in ${rclients//,/ }; do echo $i; done | sort -u)
- clients="$SINGLECLIENT $HOSTNAME $RCLIENTS"
+ export CLIENT1=${CLIENT1:-$HOSTNAME}
+ export SINGLECLIENT=$CLIENT1
+
+ clients="$SINGLECLIENT $HOSTNAME $RCLIENTS"
# Sanity check: exclude the dup entries from CLIENTS
# for those configs which has SINGLCLIENT set to local client
clients=$(for i in $clients; do echo $i; done | sort -u)
- CLIENTS=$(comma_list $clients)
+ export CLIENTS=$(comma_list $clients)
local -a remoteclients=($RCLIENTS)
for ((i=0; $i<${#remoteclients[@]}; i++)); do
varname=CLIENT$((i + 2))
- eval $varname=${remoteclients[i]}
+ eval export $varname=${remoteclients[i]}
done
- CLIENTCOUNT=$((${#remoteclients[@]} + 1))
+ export CLIENTCOUNT=$((${#remoteclients[@]} + 1))
}
get_random_entry () {
# Only check/report network health if get_param isn't reported, since
# *clearly* the network is working if get_param returned something.
if (( $(grep -c catastro $health) != $(wc -w <<< ${nodes//,/ }) )); then
- for node in ${nodes//,/}; do
+ for node in ${nodes//,/ }; do
check_network $node 5
done
fi
local facets="$facet"
if [ "$FAILURE_MODE" = HARD ]; then
- facets=$(facets_on_host $(facet_active_host $facet))
+ facets=$(for f in ${facet//,/ }; do
+ facets_on_host $(facet_active_host $f) | tr "," "\n"
+ done | sort -u | paste -sd , )
fi
for facet in ${facets//,/ }; do
local dirstripe_index=${DIRSTRIPE_INDEX:-$((base % $MDSCOUNT))}
local OPTIND=1
+ (( $MDS1_VERSION > $(version_code 2.15.0) )) &&
+ hash_name+=("crush2")
+
while getopts "c:H:i:p" opt; do
case $opt in
c) dirstripe_count=$OPTARG;;