From 7130551d63c155534ab62cb4942b7c88556daad4 Mon Sep 17 00:00:00 2001 From: Mikhail Pershin Date: Fri, 16 Oct 2015 14:04:22 +0300 Subject: [PATCH] LU-5604 tests: fix usage of drop_ldlm_reply() in tests The OBD_FAIL_LDLM_REPLY is not used to drop replies on MDT, OST and MGS anymore. But it is still used in some tests via drop_ldlm_reply() wrapper. Patch renames drop_ldlm_reply() to the drop_mdt_ldlm_reply() since it was used only for MDT. Tests were fixed also to use MDT-specific OBD_FAIL_MDS_REPLY_NET code. recovery-small.sh: test_53, test_66, test_113 and test_133 replay-dual.sh: test_19 replay-single.sh: test_73b Test 66 in recovery-small also was fixed to be aware of DNE. Tests 66 and 10c in recovery-small.sh were fixed to use 'conn_uuid' param instead of 'mds_conn_uuid' which doesn't exists now. Signed-off-by: Mikhail Pershin Change-Id: I90f7410cffcd504b3ff37728df1522693e6115cf Reviewed-on: https://review.whamcloud.com/16846 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Jian Yu Reviewed-by: Oleg Drokin --- lustre/mdt/mdt_open.c | 3 +- lustre/tests/recovery-small.sh | 76 +++++++++++++++++++++--------------------- lustre/tests/replay-dual.sh | 2 +- lustre/tests/replay-single.sh | 4 +-- lustre/tests/test-framework.sh | 16 ++++----- 5 files changed, 50 insertions(+), 51 deletions(-) diff --git a/lustre/mdt/mdt_open.c b/lustre/mdt/mdt_open.c index a91bbf1..8555c72 100644 --- a/lustre/mdt/mdt_open.c +++ b/lustre/mdt/mdt_open.c @@ -598,9 +598,8 @@ static int mdt_finish_open(struct mdt_thread_info *info, RETURN(-ENOTDIR); if (OBD_FAIL_CHECK_RESET(OBD_FAIL_MDS_OPEN_CREATE, - OBD_FAIL_LDLM_REPLY | OBD_FAIL_ONCE)) { + OBD_FAIL_MDS_LDLM_REPLY_NET | OBD_FAIL_ONCE)) RETURN(-EAGAIN); - } mfd = NULL; if (lustre_msg_get_flags(req->rq_reqmsg) & MSG_RESENT) { diff --git a/lustre/tests/recovery-small.sh b/lustre/tests/recovery-small.sh index 4ae6503..02fc696 100755 --- a/lustre/tests/recovery-small.sh +++ b/lustre/tests/recovery-small.sh @@ -211,7 +211,7 @@ test_10c() { awk '{sub("_UUID", "", $2); print $2;}') #assume one client mdccli=$($LCTL dl | grep "${mdtname}-mdc" | awk '{print $4;}') - conn_uuid=$($LCTL get_param -n mdc.${mdccli}.mds_conn_uuid) + conn_uuid=$($LCTL get_param -n mdc.${mdccli}.conn_uuid) mdcpath="mdc.${mdccli}.import=connection=${conn_uuid}" drop_bl_callback_once "chmod 0777 ${workdir}" & @@ -1288,7 +1288,7 @@ run_test 52 "failover OST under load" # test of open reconstruct test_53() { touch $DIR/$tfile - drop_ldlm_reply "openfile -f O_RDWR:O_CREAT -m 0755 $DIR/$tfile" ||\ + drop_mdt_ldlm_reply "openfile -f O_RDWR:O_CREAT -m 0755 $DIR/$tfile" ||\ return 2 } run_test 53 "touch: drop rep" @@ -1527,39 +1527,6 @@ run_test 61 "Verify to not reuse orphan objects - bug 17025" #} #run_test 62 "Verify connection flags race - bug LU-1716" -test_66() -{ - [[ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.7.51) ]] || - { skip "Need MDS version at least 2.7.51"; return 0; } - - local list=$(comma_list $(osts_nodes)) - - # modify dir so that next revalidate would not obtain UPDATE lock - touch $DIR - - # drop 1 reply with UPDATE lock - mcreate $DIR/$tfile || error "mcreate failed: $?" - drop_ldlm_reply_once "stat $DIR/$tfile" & - sleep 2 - - # make the re-sent lock to sleep -#define OBD_FAIL_MDS_RESEND 0x136 - do_nodes $list $LCTL set_param fail_loc=0x80000136 - - #initiate the re-connect & re-send - local mdccli=$($LCTL dl | awk '/-mdc-/ {print $4;}') - local conn_uuid=$($LCTL get_param -n mdc.${mdccli}.mds_conn_uuid) - $LCTL set_param "mdc.${mdccli}.import=connection=${conn_uuid}" - sleep 2 - - #initiate the client eviction while enqueue re-send is in progress - mds_evict_client - - client_reconnect - wait -} -run_test 66 "lock enqueue re-send vs client eviction" - test_65() { mount_client $DIR2 @@ -1593,6 +1560,39 @@ test_65() { } run_test 65 "lock enqueue for destroyed export" +test_66() +{ + [[ $(lustre_version_code $SINGLEMDS) -ge $(version_code 2.7.51) ]] || + { skip "Need MDS version at least 2.7.51"; return 0; } + + local list=$(comma_list $(osts_nodes)) + + # modify dir so that next revalidate would not obtain UPDATE lock + touch $DIR + + # drop 1 reply with UPDATE lock + mcreate $DIR/$tfile || error "mcreate failed: $?" + drop_mdt_ldlm_reply_once "stat $DIR/$tfile" & + sleep 2 + + # make the re-sent lock to sleep +#define OBD_FAIL_MDS_RESEND 0x136 + do_nodes $list $LCTL set_param fail_loc=0x80000136 + + #initiate the re-connect & re-send + local mdccli=$($LCTL dl | awk '/-MDT0000-mdc-/ {print $4;}') + local conn_uuid=$($LCTL get_param -n mdc.${mdccli}.conn_uuid) + $LCTL set_param "mdc.${mdccli}.import=connection=${conn_uuid}" + sleep 2 + + #initiate the client eviction while enqueue re-send is in progress + mds_evict_client + + client_reconnect + wait +} +run_test 66 "lock enqueue re-send vs client eviction" + test_67() { #define OBD_FAIL_PTLRPC_CONNECT_RACE 0x531 @@ -2505,7 +2505,7 @@ test_113() { # drop 1 reply with UPDATE lock, # resend should not create 2nd lock on server mcreate $DIR/$tfile || error "mcreate failed: $?" - drop_ldlm_reply_once "stat $DIR/$tfile" || error "stat failed: $?" + drop_mdt_ldlm_reply_once "stat $DIR/$tfile" || error "stat failed: $?" # 2 BL AST will be sent to client, both must find the same lock, # race them to not get EINVAL for 2nd BL AST @@ -2697,8 +2697,8 @@ test_133() { multiop_bg_pause $DIR/$tfile O_jc || return 1 PID=$! - #define OBD_FAIL_LDLM_REPLY 0x30c - do_nodes $list $LCTL set_param fail_loc=0x8000030c + #define OBD_FAIL_MDS_LDLM_REPLY_NET 0x157 + do_nodes $list $LCTL set_param fail_loc=0x80000157 kill -USR1 $PID echo "waiting for multiop $PID" wait $PID || return 2 diff --git a/lustre/tests/replay-dual.sh b/lustre/tests/replay-dual.sh index e9c6be9..47479fb 100755 --- a/lustre/tests/replay-dual.sh +++ b/lustre/tests/replay-dual.sh @@ -467,7 +467,7 @@ run_test 18 "ldlm_handle_enqueue succeeds on evicted export (3822)" test_19() { # Bug 10991 - resend of open request does not fail assertion. replay_barrier $SINGLEMDS - drop_ldlm_reply "createmany -o $DIR/$tfile 1" || return 1 + drop_mdt_ldlm_reply "createmany -o $DIR/$tfile 1" || return 1 fail $SINGLEMDS checkstat $DIR2/${tfile}0 || return 2 rm $DIR/${tfile}0 || return 3 diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh index 41909dc..4ce89a2 100755 --- a/lustre/tests/replay-single.sh +++ b/lustre/tests/replay-single.sh @@ -2554,8 +2554,8 @@ test_73b() { rm -f $DIR/$tfile replay_barrier $SINGLEMDS - #define OBD_FAIL_LDLM_REPLY 0x30c - do_facet $SINGLEMDS "lctl set_param fail_loc=0x8000030c" + #define OBD_FAIL_MDS_LDLM_REPLY_NET 0x157 + do_facet $SINGLEMDS "lctl set_param fail_loc=0x80000157" fail $SINGLEMDS kill -USR1 $pid wait $pid || error "multiop pid failed" diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index d2d2fc7..dc3649f 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -5617,11 +5617,11 @@ drop_bl_callback() { return $rc } -drop_ldlm_reply() { -#define OBD_FAIL_LDLM_REPLY 0x30c +drop_mdt_ldlm_reply() { +#define OBD_FAIL_MDS_LDLM_REPLY_NET 0x157 RC=0 - local list=$(comma_list $(mdts_nodes) $(osts_nodes)) - do_nodes $list lctl set_param fail_loc=0x30c + local list=$(comma_list $(mdts_nodes)) + do_nodes $list lctl set_param fail_loc=0x157 do_facet client "$@" || RC=$? @@ -5629,11 +5629,11 @@ drop_ldlm_reply() { return $RC } -drop_ldlm_reply_once() { -#define OBD_FAIL_LDLM_REPLY 0x30c +drop_mdt_ldlm_reply_once() { +#define OBD_FAIL_MDS_LDLM_REPLY_NET 0x157 RC=0 - local list=$(comma_list $(mdts_nodes) $(osts_nodes)) - do_nodes $list lctl set_param fail_loc=0x8000030c + local list=$(comma_list $(mdts_nodes)) + do_nodes $list lctl set_param fail_loc=0x80000157 do_facet client "$@" || RC=$? -- 1.8.3.1