3 # client failure does not affect other clients
5 # Start load on clients (each client works on it's own directory).
6 # At defined (5-10 minutes) interval fail one random client and then fail mds.
7 # Reintegrate failed client after recovery completed,
8 # application errors are allowed for that client but not on other clients.
9 # 10 minute intervals and verify that no application errors occur.
11 # Test runs one of CLIENT_LOAD progs on remote clients.
16 LUSTRE=${LUSTRE:-$(dirname $0)/..}
17 . $LUSTRE/tests/test-framework.sh
18 . $LUSTRE/tests/recovery-scale-lib.sh
22 # bug number for skipped test:
23 ALWAYS_EXCEPT="$RECOVERY_RANDOM_SCALE_EXCEPT "
24 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
28 remote_mds_nodsh && skip_env "remote MDS with nodsh"
29 remote_ost_nodsh && skip_env "remote OST with nodsh"
31 if (( CLIENTCOUNT < 3 )); then
32 skip_env "need three or more clients"
35 if [[ -z "$SHARED_DIRECTORY" ]] || ! check_shared_dir "$SHARED_DIRECTORY"; then
36 skip_env "SHARED_DIRECTORY should be specified with a shared directory \
37 which is accessible on all of the nodes"
40 if [[ "$FAILURE_MODE" == "SOFT" ]]; then
41 log "WARNING: $0 is not functional with FAILURE_MODE = SOFT, bz22797"
44 # Application failures are allowed for the failed client
45 # but not for other clients.
52 for facet in ${MDTS//,/ } ${FAILED_CLIENTS//,/ }; do
53 var=$(node_var_name $facet)_nums
56 echo "$facet failed over $val times"
61 check_and_setup_lustre
62 rm -rf $DIR/[Rdfs][0-9]*
65 check_progs_installed $NODES_TO_USE "${CLIENT_LOADS[@]}"
67 MAX_RECOV_TIME=$(max_recovery_time)
68 MDTS=$(get_facets MDS)
69 OSTS=$(get_facets OST)
71 # Fail a random client and then failover a random MDS.
72 test_fail_client_mds() {
78 stack_trap summary_and_cleanup EXIT INT
80 # start vmstat on OSS nodes
82 start_vmstat $(comma_list $(osts_nodes)) $VMSTAT_PID_FILE
86 start_client_loads $NODES_TO_USE
88 echo client loads pids:
89 do_nodesv $NODES_TO_USE "cat $LOAD_PID_FILE" || exit 3
95 local start_ts=$(date +%s)
96 local current_ts=$start_ts
98 while [ $ELAPSED -lt $DURATION -a ! -e $END_RUN_FILE ]; do
99 # In order to perform the
100 # expected number of failovers, we need to account the
102 # 1) the time that has elapsed during the client load checking
103 # 2) time takes for failover
104 it_time_start=$(date +%s)
106 fail_client=$(get_random_entry $NODES_TO_USE)
107 client_var=$(node_var_name $fail_client)_nums
109 # store the list of failed clients
110 # lists are comma separated
111 FAILED_CLIENTS=$(expand_list $FAILED_CLIENTS $fail_client)
113 serverfacet=$(get_random_entry $MDTS)
114 var=$(node_var_name $serverfacet)_nums
116 # Check that our client loads are still running. If any have
117 # died, that means they have died outside of recovery, which
119 log "==== Checking clients loads BEFORE failover -- failure NOT OK \
120 ELAPSED=$ELAPSED DURATION=$DURATION \
121 PERIOD=$SERVER_FAILOVER_PERIOD"
122 check_client_loads $NODES_TO_USE || exit 4
124 log "FAIL CLIENT $fail_client..."
125 shutdown_client $fail_client
127 log "Starting failover on $serverfacet"
128 facet_failover "$serverfacet" || exit 1
130 if ! wait_recovery_complete $serverfacet; then
131 echo "$serverfacet recovery is not completed!"
135 boot_node $fail_client
136 echo "Reintegrating $fail_client"
137 zconf_mount $fail_client $MOUNT || exit $?
138 client_up $fail_client || exit $?
140 # Increment the number of failovers
143 val=$((${!client_var} + 1))
144 eval $client_var=$val
146 # load script on failed clients could create END_RUN_FILE
147 # We shuold remove it and ignore the failure if this
148 # file contains the failed client only.
149 # We can not use ERRORS_OK when start all loads at the start of
150 # this script because the application errors allowed for random
151 # failed client only, but not for all clients.
152 if [ -e $END_RUN_FILE ]; then
154 read end_run_node < $END_RUN_FILE
155 [[ $end_run_node = $fail_client ]] &&
156 rm -f $END_RUN_FILE || exit 13
159 restart_client_loads $fail_client $ERRORS_OK || exit $?
161 # Check that not failed clients loads are still running.
162 # No application failures should occur on clients that were
164 log "==== Checking clients loads AFTER failed client reintegrated \
166 if ! ERRORS_OK= check_client_loads \
167 $(exclude_items_from_list $NODES_TO_USE $fail_client); then
168 log "Client load failed. Exiting..."
172 current_ts=$(date +%s)
173 ELAPSED=$((current_ts - start_ts))
174 sleep=$((SERVER_FAILOVER_PERIOD - (current_ts - it_time_start)))
176 # Keep counting the number of iterations when
177 # time spent to failover and two client loads check exceeded
178 # the value ( SERVER_FAILOVER_PERIOD - MINSLEEP ).
179 if [ $sleep -lt $MINSLEEP ]; then
180 reqfail=$((reqfail + 1))
181 log "WARNING: failover, client reintegration and \
182 check_client_loads time exceeded SERVER_FAILOVER_PERIOD - MINSLEEP!
183 Failed to load the filesystem with I/O for a minimum period of \
184 $MINSLEEP $reqfail times ( REQFAIL=$REQFAIL ).
185 This iteration, the load was only applied for sleep=$sleep seconds.
186 Estimated max recovery time : $MAX_RECOV_TIME
187 Probably the hardware is taking excessively long time to boot.
188 Try to increase SERVER_FAILOVER_PERIOD (current is $SERVER_FAILOVER_PERIOD), \
190 [ $reqfail -gt $REQFAIL ] && exit 6
193 log "Number of failovers:
194 $(numfailovers) and counting..."
196 [ $((ELAPSED + sleep)) -ge $DURATION ] && break
198 if [ $sleep -gt 0 ]; then
199 echo "sleeping $sleep seconds... "
205 run_test fail_client_mds "fail client, then failover MDS"
207 zconf_mount $HOSTNAME $MOUNT || error "mount $MOUNT on $HOSTNAME failed"
208 client_up || error "start client on $HOSTNAME failed"
210 complete_test $SECONDS
211 check_and_cleanup_lustre