3 # All pairwise combinations of node failures.
6 # Author: Chris Cooper <ccooper@clusterfs.com>
8 # Script fails pair of nodes:
9 # -- in parallel by default
10 # -- in series if SERIAL is set
15 LUSTRE=${LUSTRE:-$(dirname $0)/..}
16 . $LUSTRE/tests/test-framework.sh
20 # bug number for skipped test:
21 ALWAYS_EXCEPT="$RECOVERY_DOUBLE_SCALE_EXCEPT"
22 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
26 remote_mds_nodsh && skip_env "remote MDS with nodsh"
27 remote_ost_nodsh && skip_env "remote OST with nodsh"
29 [$CLIENTCOUNT -lt 3 ] &&
30 skip_env "need three or more clients"
32 if [ -z "$SHARED_DIRECTORY" ] || ! check_shared_dir $SHARED_DIRECTORY; then
33 skip_env "SHARED_DIRECTORY should be specified with a shared directory \
34 which is accessable on all of the nodes"
38 [[ $FAILURE_MODE = SOFT ]] && \
39 log "WARNING: $0 is not functional with FAILURE_MODE = SOFT, bz22797"
41 # Set SERIAL to serialize the failure through a recovery of the first failure.
45 [ "$SERIAL" ] && ERRORS_OK=""
47 FAILOVER_PERIOD=${FAILOVER_PERIOD:-$((60 * 5))} # 5 minutes
49 END_RUN_FILE=${END_RUN_FILE:-$SHARED_DIRECTORY/end_run_file}
50 LOAD_PID_FILE=${LOAD_PID_FILE:-$TMP/client-load.pid}
52 reboot_recover_node () {
53 # item var contains a pair of clients if nodetype=clients
54 # I would prefer to have a list here
59 # MDS, OST item contains the facet
61 MDS|OST ) facet_failover $item
62 [ "$SERIAL" ] && wait_recovery_complete $item || true
64 clients) for c in ${item//,/ }; do
65 # make sure the client loads die
66 stop_process $c $LOAD_PID_FILE
69 echo "Reintegrating $c"
70 zconf_mount $c $MOUNT ||
71 error "mount $MOUNT on $c failed"
72 client_up $c || error "start client on $c failed"
74 start_client_loads $item
76 * ) echo "ERROR: invalid nodetype=$nodetype." \
77 "Must be one of 'MDS', 'OST', or 'clients'."
84 local excluded=${2:-""}
90 clients) list=$NODES_TO_USE;;
91 * ) echo "ERROR: invalid type=$type." \
92 "Must be one of 'MDS', 'OST', or 'clients'."
96 [ "$excluded" ] && list=$(exclude_items_from_list $list $excluded)
98 if [ ! "$(echo $list)" ]; then
103 local item=$(get_random_entry $list)
104 if [ "$type" = "clients" ]; then
105 item="$item $(get_random_entry $(exclude_items_from_list $list $item))"
106 item=$(comma_list $item)
113 # for the two nodetypes specified, chooses a random node(s) from each
114 # class, reboots the nodes sequentially, and then restarts lustre on
121 local client_nodes=""
128 ==== START === $title"
130 item1=$(get_item_type $type1)
132 { echo "type1=$type1 item1 is empty" && return 0; }
133 item2=$(get_item_type $type2 $item1)
135 { echo "type1=$type1 item1=$item1 type2=$type2 item2=$item2 is empty" \
138 # Check that our client loads are still running. If any have died,
139 # that means they have died outside of recovery, which is unacceptable.
140 log "==== Checking the clients loads BEFORE failover -- failure NOT OK"
141 # FIXME. need print summary on exit
142 check_client_loads $NODES_TO_USE || exit $?
144 log "Done checking client loads. Failing type1=$type1 item1=$item1 ... "
145 reboot_recover_node $item1 $type1 || exit $?
147 # Hendrix test17 description:
148 # Introduce a failure, wait at
149 # least 5 minutes (for recovery),
151 # failure, and wait another 5
154 # reboot_recover_node waits recovery in according to
156 # We have a "double failures" if SERIAL is not set,
157 # do not need a sleep between failures for "double failures"
159 log " Failing type2=$type2 item2=$item2 ... "
160 reboot_recover_node $item2 $type2 || exit $?
162 # Client loads are allowed to die while in recovery, so we just
164 log "==== Checking the clients loads AFTER failovers -- ERRORS_OK=$ERRORS_OK"
165 restart_client_loads $NODES_TO_USE $ERRORS_OK || exit $?
166 log "Done checking / re-starting client loads. PASS"
170 summary_and_cleanup () {
174 CURRENT_TS=$(date +%s)
175 ELAPSED=$((CURRENT_TS - START_TS))
177 # Having not empty END_RUN_FILE means the failed loads only
178 if [ -s $END_RUN_FILE ]; then
179 print_end_run_file $END_RUN_FILE
183 echo $(date +'%F %H:%M:%S') Terminating clients loads ...
184 echo "$0" >> $END_RUN_FILE
186 [ $rc -eq 0 ] || result=FAIL
189 Server failover period: $FAILOVER_PERIOD seconds
190 Exited after: $ELAPSED seconds
191 Status: $result: rc=$rc"
193 # stop the client loads
194 stop_client_loads $NODES_TO_USE $LOAD_PID_FILE
196 if [ $rc -ne 0 ]; then
197 # we are interested in only on failed clients and servers
198 local failedclients=$(cat $END_RUN_FILE | grep -v $0)
199 gather_logs $(comma_list $(all_server_nodes) $failedclients)
205 ################################## Main Flow ###################################
206 check_and_setup_lustre
207 rm -rf $DIR/[Rdfs][0-9]*
209 check_timeout || exit 1
211 # The test node needs to be insulated from a lustre failure as much as possible,
212 # so not even loading the lustre modules is ideal.
214 # -- remove hostname from clients list
215 zconf_umount $HOSTNAME $MOUNT
216 NODES_TO_USE=${NODES_TO_USE:-$CLIENTS}
217 NODES_TO_USE=$(exclude_items_from_list $NODES_TO_USE $HOSTNAME)
219 check_progs_installed $NODES_TO_USE "${CLIENT_LOADS[@]}"
221 MDTS=$(get_facets MDS)
222 OSTS=$(get_facets OST)
228 # Every pairwise combination of client failures (2 clients),
229 # MDS failure, and OST failure will be tested.
230 test_pairwise_fail() {
231 trap summary_and_cleanup EXIT TERM INT
233 # Start client loads.
235 start_client_loads $NODES_TO_USE
237 echo clients load pids:
238 do_nodesv $NODES_TO_USE "cat $LOAD_PID_FILE" || exit 3
240 # FIXME: Do we want to have an initial sleep period where the clients
241 # just run before introducing a failure?
242 sleep $FAILOVER_PERIOD
245 failover_pair MDS OST "test 1: failover MDS, then OST =========="
246 sleep $FAILOVER_PERIOD
249 failover_pair MDS clients "test 2: failover MDS, then 2 clients ===="
250 sleep $FAILOVER_PERIOD
253 if [ $MDSCOUNT -gt 1 ]; then
254 failover_pair MDS MDS "test 3: failover MDS, then another MDS =="
255 sleep $FAILOVER_PERIOD
257 skip_env "has less than 2 MDTs, test 3 skipped"
261 if [ $OSTCOUNT -gt 1 ]; then
262 failover_pair OST OST "test 4: failover OST, then another OST =="
263 sleep $FAILOVER_PERIOD
265 skip_env "has less than 2 OSTs, test 4 skipped"
269 failover_pair OST clients "test 5: failover OST, then 2 clients ===="
270 sleep $FAILOVER_PERIOD
273 failover_pair OST MDS "test 6: failover OST, then MDS =========="
274 sleep $FAILOVER_PERIOD
277 failover_pair clients MDS "test 7: failover 2 clients, then MDS ===="
278 sleep $FAILOVER_PERIOD
281 failover_pair clients OST "test 8: failover 2 clients, then OST ===="
282 sleep $FAILOVER_PERIOD
285 if [ $CLIENTCOUNT -gt 4 ]; then
286 failover_pair clients clients \
287 "test 9: failover 2 clients, then 2 different clients =="
288 sleep $FAILOVER_PERIOD
290 skip_env "has less than 5 Clients, test 9 skipped"
293 log "==== Checking the clients loads AFTER all failovers -- failure NOT OK"
294 if ! check_client_loads $NODES_TO_USE; then
295 log "Client load failed after failover. Exiting..."
301 run_test pairwise_fail "pairwise combination of clients, MDS, and OST failures"
303 zconf_mount $HOSTNAME $MOUNT || error "mount $MOUNT on $HOSTNAME failed"
304 client_up || error "start client on $HOSTNAME failed"
307 check_and_cleanup_lustre