+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
+
+ shutdown_facet mds1
+ reboot_facet mds1
+
+ # prepare for MDS failover
+ change_active mds1
+ reboot_facet mds1
+
+ clients_up &
+ DFPID=$!
+ sleep 5
+
+ shutdown_facet ost1
+
+ echo "Reintegrating OST"
+ reboot_facet ost1
+ wait_for_facet ost1
+ start_ost 1 || return 2
+
+ shutdown_facet mds2
+ reboot_facet mds2
+
+ # prepare for MDS failover
+ change_active mds2
+ reboot_facet mds2
+
+ wait_for_facet mds1
+ start_mdt 1 || return $?
+
+ wait_for_facet mds2
+ start_mdt 2 || return $?
+
+ #Check FS
+ wait $DFPID
+ clients_recover_osts ost1
+ echo "Verify reintegration"
+ clients_up || return 1
+}
+run_test 10 "Tenth Failure Mode: MDT0/OST/MDT1 `date`"
+###################################################
+
+############### Seventh Failure Mode ###############
+test_11() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
+ echo "Verify Lustre filesystem is up and running"
+ [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
+
+ #MDS Portion
+ fail mds1
+ #Check FS
+
+ echo "Test Lustre stability after MDS failover"
+ clients_up
+
+ #CLIENT Portion
+ echo "Failing 2 CLIENTS"
+ fail_clients 2
+
+ #Check FS
+ echo "Test Lustre stability after CLIENT failure"
+ clients_up
+
+ #Reintegration
+ echo "Reintegrating CLIENTS"
+ reintegrate_clients || return 1
+
+ fail mds2
+
+ clients_up || return 3
+ sleep 2 # give it a little time for fully recovered before next test
+}
+run_test 11 "Eleventh Failure Mode: MDS0/CLIENT/MDS1 `date`"
+###################################################
+
+test_12() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
+ echo "Verify Lustre filesystem is up and running"
+ [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
+
+ #MDS Portion
+ fail mds1,mds2
+ clients_up
+
+ #OSS Portion
+ fail ost1,ost2
+ clients_up
+
+ #CLIENT Portion
+ echo "Failing 2 CLIENTS"
+ fail_clients 2
+
+ #Check FS
+ echo "Test Lustre stability after CLIENT failure"
+ clients_up
+
+ #Reintegration
+ echo "Reintegrating CLIENTS"
+ reintegrate_clients || return 1
+
+ clients_up || return 3
+ sleep 2 # give it a little time for fully recovered before next test
+}
+run_test 12 "Twelve Failure Mode: MDS0,MDS1/OST0, OST1/CLIENTS `date`"
+###################################################
+
+test_13() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
+ echo "Verify Lustre filesystem is up and running"
+ [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
+
+ #MDS Portion
+ fail mds1,mds2
+ clients_up
+
+ #CLIENT Portion
+ echo "Failing 2 CLIENTS"
+ fail_clients 2
+
+ #Check FS
+ echo "Test Lustre stability after CLIENT failure"
+ clients_up
+
+ #Reintegration
+ echo "Reintegrating CLIENTS"
+ reintegrate_clients || return 1
+
+ clients_up || return 3
+ sleep 2 # give it a little time for fully recovered before next test
+
+ #OSS Portion
+ fail ost1,ost2
+ clients_up || return 4
+}
+run_test 13 "Thirteen Failure Mode: MDS0,MDS1/CLIENTS/OST0,OST1 `date`"
+###################################################
+
+test_14() {
+ [ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs" && return
+ echo "Verify Lustre filesystem is up and running"
+ [ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"
+
+ #OST Portion
+ fail ost1,ost2
+ clients_up
+
+ #CLIENT Portion
+ echo "Failing 2 CLIENTS"
+ fail_clients 2
+
+ #Check FS
+ echo "Test Lustre stability after CLIENT failure"
+ clients_up
+
+ #Reintegration
+ echo "Reintegrating CLIENTS"
+ reintegrate_clients || return 1
+
+ clients_up || return 3
+ sleep 2 # give it a little time for fully recovered before next test
+
+ #OSS Portion
+ fail mds1,mds2
+ clients_up || return 4
+}
+run_test 14 "Fourteen Failure Mode: OST0,OST1/CLIENTS/MDS0,MDS1 `date`"
+###################################################
+
+test_15() {