next_id2=$(do_facet mds lctl get_param -n osc.$mdtosc.prealloc_next_id)
echo "before recovery: last_id = $last_id2, next_id = $next_id2"
+ # if test uses shutdown_facet && reboot_facet instead of facet_failover ()
+ # it has to take care about the affected facets, bug20407
+ local affected_mds=$(affected_facets mds)
+ local affected_ost1=$(affected_facets ost1)
+
shutdown_facet mds
shutdown_facet ost1
reboot_facet mds
- change_active mds
- wait_for_facet mds
- mount_facet mds || error "Restart of mds failed"
+ change_active $affected_mds
+ wait_for_facet $affected_mds
+ mount_facets $affected_mds || error "Restart of mds failed"
reboot_facet ost1
- change_active ost1
- wait_for_facet ost1
- mount_facet ost1 || error "Restart of ost1 failed"
+ change_active $affected_ost1
+ wait_for_facet $affected_ost1
+ mount_facets $affected_ost1 || error "Restart of ost1 failed"
clients_up
}
# Facet functions
+mount_facets () {
+ local facets=${1:-$(get_facets)}
+ local facet
+
+ for facet in ${facets//,/ }; do
+ mount_facet $facet || error "Restart of $facet failed!"
+ done
+}
+
mount_facet() {
local facet=$1
shift
echo $(comma_list $affected)
}
+facet_up () {
+ local facet=$1
+ local host=${2:-$(facet_host $facet)}
+
+ local label=$(convert_facet2label $facet)
+ do_node $host lctl dl | awk '{print $4}' | grep -q $label
+}
+
+facets_up_on_host () {
+ local host=$1
+ local facets=$(facets_on_host $host)
+ local affected_up
+
+ for facet in ${facets//,/ }; do
+ if $(facet_up $facet $host); then
+ affected_up="$affected_up $facet"
+ fi
+ done
+
+ echo $(comma_list $affected_up)
+}
+
shutdown_facet() {
local facet=$1
rm $MOUNT/recon
}
+affected_facets () {
+ local facet=$1
+
+ local host=$(facet_active_host $facet)
+ local affected=$facet
+
+ if [ "$FAILURE_MODE" = HARD ]; then
+ affected=$(facets_up_on_host $host)
+ fi
+ echo $affected
+}
+
facet_failover() {
local facet=$1
local sleep_time=$2
echo "Failing $facet on node $host"
- local affected=$facet
-
- if [ "$FAILURE_MODE" = HARD ]; then
- affected=$(facets_on_host $host)
- fi
+ local affected=$(affected_facets $facet)
shutdown_facet $facet
if ! combined_mgs_mds && list_member $affected mgs; then
mount_facet mgs || error "Restart of mgs failed"
fi
- # FIXME; has to be changed to mount all facets concurrently
+ # FIXME; has to be changed to mount all facets concurrently
affected=$(exclude_items_from_list $affected mgs)
- for facet in ${affected//,/ }; do
- mount_facet $facet || error "Restart of $facet on node $host failed!"
- done
+ mount_facets $affected
}
obd_name() {