X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftests%2Frecovery-small.sh;h=125051b457c88c8628bd66ccd8b14f198341b18f;hp=02fc6962dede68b0af27d0cd900cbed504fe8aa4;hb=20472b4070495c1bb2796ce2682aefdf2fe8a3f6;hpb=7130551d63c155534ab62cb4942b7c88556daad4;ds=sidebyside diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh index 02fc696..125051b 100755 --- a/lustre/tests/recovery-small.sh +++ b/lustre/tests/recovery-small.sh @@ -2,8 +2,9 @@ set -e -# bug 5493 LU2034 -ALWAYS_EXCEPT="52 $RECOVERY_SMALL_EXCEPT" +# bug number for skipped test: +ALWAYS_EXCEPT="$RECOVERY_SMALL_EXCEPT" +# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! export MULTIOP=${MULTIOP:-multiop} PTLDEBUG=${PTLDEBUG:--1} @@ -1756,26 +1757,45 @@ test_100() } run_test 100 "IR: Make sure normal recovery still works w/o IR" -test_101() +test_101a() { - do_facet mgs $LCTL list_param mgs.*.ir_timeout || - { skip "MGS without IR support"; return 0; } + do_facet mgs $LCTL list_param mgs.*.ir_timeout || + skip "MGS without IR support" - set_ir_status full + set_ir_status full - local OST1_IMP=$(get_osc_import_name client ost1) + local ost1_imp=$(get_osc_import_name client ost1) - # disable pinger recovery - lctl set_param -n osc.$OST1_IMP.pinger_recov=0 + # disable pinger recovery + lctl set_param -n osc.$ost1_imp.pinger_recov=0 + stack_trap "$LCTL set_param -n osc.$ost1_imp.pinger_recov=1" EXIT - fail ost1 + fail ost1 - target_instance_match ost1 || error "instance mismatch" - nidtbl_versions_match || error "version must match" + target_instance_match ost1 || error "instance mismatch" + nidtbl_versions_match || error "version must match" +} +run_test 101a "IR: Make sure IR works w/o normal recovery" - lctl set_param -n osc.$OST1_IMP.pinger_recov=1 +test_101b() +{ + do_facet mgs $LCTL list_param mgs.*.ir_timeout || + skip "MGS without IR support" + + set_ir_status full + + local ost1_imp=$(get_osc_import_name client ost1) + +#define OBD_FAIL_OST_PREPARE_DELAY 0x247 + do_facet ost1 "$LCTL set_param fail_loc=0x247" + # disable pinger recovery + $LCTL set_param -n osc.$ost1_imp.pinger_recov=0 + stack_trap "$LCTL set_param -n osc.$ost1_imp.pinger_recov=1" EXIT + +#OST may return EAGAIN if it is not configured yet + fail ost1 } -run_test 101 "IR: Make sure IR works w/o normal recovery" +run_test 101b "IR: Make sure IR works w/o normal recovery and proceed EAGAIN" test_102() { @@ -2739,6 +2759,76 @@ test_134() { } run_test 134 "race between failover and search for reply data free slot" +test_135() { + [ $MDS1_VERSION -lt $(version_code 2.12.51) ] && + skip "Need MDS version at least 2.12.51" + + mkdir -p $DIR/$tdir + $LFS setstripe -E 1M -L mdt $DIR/$tdir + # to have parent dir write lock before open/resend + touch $DIR/$tdir/$tfile + #define OBD_FAIL_MDS_LDLM_REPLY_NET 0x157 + do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param fail_loc=0x80000157 + openfile -f O_RDWR:O_CREAT -m 0755 $DIR/$tdir/$tfile || + error "Failed to open DOM file" +} +run_test 135 "DOM: open/create resend to return size" + +test_136() { + remote_mds_nodsh && skip "remote MDS with nodsh" && return + [[ $MDS1_VERSION -ge $(version_code 2.12.52) ]] || + skip "Need MDS version at least 2.12.52" + + local mdts=$(comma_list $(mdts_nodes)) + local MDT0=$(facet_svc $SINGLEMDS) + + local clog=$(do_facet mds1 $LCTL --device $MDT0 changelog_register -n) + [ -n "$clog" ] || error "changelog_register failed" + cl_mask=$(do_facet mds1 $LCTL get_param \ + mdd.$MDT0.changelog_mask -n) + changelog_chmask "ALL" + + # generate some changelog records to accumulate + test_mkdir -i 0 -c 0 $DIR/$tdir || error "mkdir $tdir failed" + createmany -m $DIR/$tdir/$tfile 10000 || + error "create $DIR/$tdir/$tfile failed" + + local size1=$(do_facet $SINGLEMDS \ + $LCTL get_param -n mdd.$MDT0.changelog_size) + echo "Changelog size $size1" + + #define OBD_FAIL_LLOG_PURGE_DELAY 0x1318 + do_nodes $mdts $LCTL set_param fail_loc=0x1318 fail_val=30 + + # launch changelog_deregister in background on MDS + do_facet mds1 "nohup $LCTL --device $MDT0 changelog_deregister $clog \ + > foo.out 2> foo.err < /dev/null &" + # give time to reach fail_loc + sleep 15 + + # fail_loc will make MDS sleep in the middle of changelog_deregister + # take this opportunity to abruptly kill MDS + FAILURE_MODE_save=$FAILURE_MODE + FAILURE_MODE=HARD + fail mds1 + FAILURE_MODE=$FAILURE_MODE_save + + do_nodes $mdts $LCTL set_param fail_loc=0x0 fail_val=0 + + local size2=$(do_facet $SINGLEMDS \ + $LCTL get_param -n mdd.$MDT0.changelog_size) + echo "Changelog size $size2" + local clog2=$(do_facet $SINGLEMDS "$LCTL get_param -n \ + mdd.$MDT0.changelog_users | grep $clog") + echo "After crash, changelog user $clog2" + + [ -n "$clog2" -o $size2 -lt $size1 ] || + error "changelog record count unchanged" + + do_facet mds1 $LCTL set_param mdd.$MDT0.changelog_mask=\'$cl_mask\' -n +} +run_test 136 "changelog_deregister leaving pending records" + complete $SECONDS check_and_cleanup_lustre exit_status