Whamcloud - gitweb
LU-5569 ptlrpc: change reverse import life cycle 50/11750/15
authorAlexey Lyashkov <alexey_lyashkov@xyratex.com>
Fri, 5 Jun 2015 15:46:48 +0000 (11:46 -0400)
committerOleg Drokin <oleg.drokin@intel.com>
Sat, 19 Sep 2015 03:50:11 +0000 (03:50 +0000)
Make reverse import on server have same life cycle for a client
import, otherwise a reverse import disconnecting on each client
reconnect open several races in request sending (AST mostly) code.

First problem is send_rpc vs class_destroy_import() race. If send RPC
(or resending) issued after class_destroy_import() function it will
fail due to generation check.

The second problem is resending via a different router (to a different nid).
The target_handle_connect() function doesn't update the connection
information for older reverse import, so if the connection information
or security flavor has changed we won't be able to deliver an RPC
from server to the client.

The third problem is that connection flags aren't updated atomically
for an import. The target_handle_connect() function connects the new
import before the message header flags are set, so if we send an RPC
in that time it may have the wrong flags.

The final fourth problem is none wakeup an older RPC if client
reconnected to ability to resend after network flap. This was
not a problem without Vitaly's "resend AST callbacks" patch
(commit 30be03b4dd59389) as it was not possible to resend RPCs.
Now, however, this problem results in failing to resend ASTs at
all, or adding long timeout to AST RPCs.

Xyratex-bug-id: MRP-2038
Signed-off-by: Alexey Lyashkov <alexey_lyashkov@xyratex.com>
Change-Id: I5dd65a0a507827d6a43683dedbbc0cee263ee0d0
Reviewed-on: http://review.whamcloud.com/11750
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Jinshan Xiong <jinshan.xiong@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
lustre/include/lustre_lib.h
lustre/include/lustre_net.h
lustre/ldlm/ldlm_lib.c
lustre/obdclass/genops.c
lustre/ptlrpc/import.c
lustre/target/tgt_lastrcvd.c
lustre/tests/recovery-small.sh

index a343280..d765fa2 100644 (file)
@@ -62,7 +62,8 @@ struct l_wait_info;
 
 #ifdef HAVE_SERVER_SUPPORT
 void target_client_add_cb(struct obd_device *obd, __u64 transno, void *cb_data,
 
 #ifdef HAVE_SERVER_SUPPORT
 void target_client_add_cb(struct obd_device *obd, __u64 transno, void *cb_data,
-                          int error);
+                         int error);
+int rev_import_init(struct obd_export *exp);
 int target_handle_connect(struct ptlrpc_request *req);
 int target_handle_disconnect(struct ptlrpc_request *req);
 void target_destroy_export(struct obd_export *exp);
 int target_handle_connect(struct ptlrpc_request *req);
 int target_handle_disconnect(struct ptlrpc_request *req);
 void target_destroy_export(struct obd_export *exp);
index b06073b..a7f37c2 100644 (file)
@@ -1081,7 +1081,6 @@ struct ptlrpc_request {
 
        /** early replies go to offset 0, regular replies go after that */
        unsigned int                     rq_reply_off;
 
        /** early replies go to offset 0, regular replies go after that */
        unsigned int                     rq_reply_off;
-
        /** @} */
 
        /** Fields that help to see if request and reply were swabbed or not */
        /** @} */
 
        /** Fields that help to see if request and reply were swabbed or not */
@@ -1512,7 +1511,7 @@ struct ptlrpc_thread {
         /**
          * service thread pid
          */
         /**
          * service thread pid
          */
-        pid_t t_pid;
+       pid_t t_pid;
         /**
          * put watchdog in the structure per thread b=14840
          */
         /**
          * put watchdog in the structure per thread b=14840
          */
@@ -2285,8 +2284,8 @@ int ptlrpc_init_import(struct obd_import *imp);
 int ptlrpc_disconnect_import(struct obd_import *imp, int noclose);
 int ptlrpc_import_recovery_state_machine(struct obd_import *imp);
 void deuuidify(char *uuid, const char *prefix, char **uuid_start,
 int ptlrpc_disconnect_import(struct obd_import *imp, int noclose);
 int ptlrpc_import_recovery_state_machine(struct obd_import *imp);
 void deuuidify(char *uuid, const char *prefix, char **uuid_start,
-               int *uuid_len);
-
+              int *uuid_len);
+void ptlrpc_import_enter_resend(struct obd_import *imp);
 /* ptlrpc/pack_generic.c */
 int ptlrpc_reconnect_import(struct obd_import *imp);
 /** @} */
 /* ptlrpc/pack_generic.c */
 int ptlrpc_reconnect_import(struct obd_import *imp);
 /** @} */
index a942d90..1d4d6b7 100644 (file)
@@ -721,13 +721,13 @@ static int target_handle_reconnect(struct lustre_handle *conn,
                                    struct obd_export *exp,
                                    struct obd_uuid *cluuid)
 {
                                    struct obd_export *exp,
                                    struct obd_uuid *cluuid)
 {
-        ENTRY;
+       struct lustre_handle *hdl;
+       ENTRY;
 
 
-        if (exp->exp_connection && exp->exp_imp_reverse) {
-                struct lustre_handle *hdl;
+       hdl = &exp->exp_imp_reverse->imp_remote_handle;
+       if (exp->exp_connection && lustre_handle_is_used(hdl)) {
                 struct obd_device *target;
 
                 struct obd_device *target;
 
-                hdl = &exp->exp_imp_reverse->imp_remote_handle;
                 target = exp->exp_obd;
 
                 /* Might be a re-connect after a partition. */
                 target = exp->exp_obd;
 
                 /* Might be a re-connect after a partition. */
@@ -797,17 +797,124 @@ static void
 check_and_start_recovery_timer(struct obd_device *obd,
                                struct ptlrpc_request *req, int new_client);
 
 check_and_start_recovery_timer(struct obd_device *obd,
                                struct ptlrpc_request *req, int new_client);
 
+/**
+ * update flags for import during reconnect process
+ */
+static int rev_import_flags_update(struct obd_import *revimp,
+                                  struct ptlrpc_request *req)
+{
+       int rc;
+       struct obd_connect_data *data;
+
+       data = req_capsule_client_get(&req->rq_pill, &RMF_CONNECT_DATA);
+
+       if (data->ocd_connect_flags & OBD_CONNECT_AT)
+               revimp->imp_msghdr_flags |= MSGHDR_AT_SUPPORT;
+       else
+               revimp->imp_msghdr_flags &= ~MSGHDR_AT_SUPPORT;
+
+       revimp->imp_msghdr_flags |= MSGHDR_CKSUM_INCOMPAT18;
+
+       rc = sptlrpc_import_sec_adapt(revimp, req->rq_svc_ctx, &req->rq_flvr);
+       if (rc) {
+               CERROR("%s: cannot get reverse import %s security: rc = %d\n",
+                       revimp->imp_client->cli_name,
+                       libcfs_id2str(req->rq_peer), rc);
+               return rc;
+       }
+
+       return 0;
+}
+
+/**
+ * Allocate a new reverse import for an export.
+ *
+ * \retval -errno in case error hit
+ * \retval 0 if reverse import correctly init
+ **/
+int rev_import_init(struct obd_export *export)
+{
+       struct obd_device *obd = export->exp_obd;
+       struct obd_import *revimp;
+
+       LASSERT(export->exp_imp_reverse == NULL);
+
+       revimp = class_new_import(obd);
+       if (revimp == NULL)
+               return -ENOMEM;
+
+       revimp->imp_remote_handle.cookie = 0ULL;
+       revimp->imp_client = &obd->obd_ldlm_client;
+       revimp->imp_dlm_fake = 1;
+
+       /* it is safe to connect import in new state as no sends possible */
+       spin_lock(&export->exp_lock);
+       export->exp_imp_reverse = revimp;
+       spin_unlock(&export->exp_lock);
+       class_import_put(revimp);
+
+       return 0;
+}
+EXPORT_SYMBOL(rev_import_init);
+
+/**
+ * Handle reconnect for an export.
+ *
+ * \param exp export to handle reconnect process
+ * \param req client reconnect request
+ *
+ * \retval -rc in case securitfy flavor can't be changed
+ * \retval 0 in case none problems
+ */
+static int rev_import_reconnect(struct obd_export *exp,
+                               struct ptlrpc_request *req)
+{
+       struct obd_import *revimp = exp->exp_imp_reverse;
+       struct lustre_handle *lh;
+       int rc;
+
+       /* avoid sending a request until import flags are changed */
+       ptlrpc_import_enter_resend(revimp);
+
+       if (revimp->imp_connection != NULL)
+               ptlrpc_connection_put(revimp->imp_connection);
+
+       /*
+        * client from recovery don't have a handle so we need to take from
+        * request. it may produce situation when wrong client connected
+        * to recovery as we trust a client uuid
+        */
+       lh = req_capsule_client_get(&req->rq_pill, &RMF_CONN);
+       revimp->imp_remote_handle = *lh;
+
+       /* unknown versions will be caught in
+        * ptlrpc_handle_server_req_in->lustre_unpack_msg() */
+       revimp->imp_msg_magic = req->rq_reqmsg->lm_magic;
+
+       revimp->imp_connection = ptlrpc_connection_addref(exp->exp_connection);
+
+       rc = rev_import_flags_update(revimp, req);
+       if (rc != 0) {
+               /* it is safe to still be in RECOVERY phase as we are not able
+                * to setup correct security flavor so requests are not able to
+                * be delivered correctly */
+               return rc;
+       }
+
+       /* resend all rpc's via new connection */
+       return ptlrpc_import_recovery_state_machine(revimp);
+}
+
 int target_handle_connect(struct ptlrpc_request *req)
 {
 int target_handle_connect(struct ptlrpc_request *req)
 {
-       struct obd_device *target = NULL, *targref = NULL;
-        struct obd_export *export = NULL;
-        struct obd_import *revimp;
-       struct obd_import *tmp_imp = NULL;
-        struct lustre_handle conn;
-        struct lustre_handle *tmp;
+       struct obd_device *target = NULL;
+       struct obd_export *export = NULL;
+       /* connect handle - filled from target_handle_reconnect in
+        * reconnect case */
+       struct lustre_handle conn;
+       struct lustre_handle *tmp;
         struct obd_uuid tgtuuid;
         struct obd_uuid cluuid;
         struct obd_uuid tgtuuid;
         struct obd_uuid cluuid;
-        struct obd_uuid remote_uuid;
         char *str;
         int rc = 0;
         char *target_start;
         char *str;
         int rc = 0;
         char *target_start;
@@ -844,6 +951,11 @@ int target_handle_connect(struct ptlrpc_request *req)
        }
 
        spin_lock(&target->obd_dev_lock);
        }
 
        spin_lock(&target->obd_dev_lock);
+       /* Make sure the target isn't cleaned up while we're here. Yes,
+        * there's still a race between the above check and our incref here.
+        * Really, class_uuid2obd should take the ref. */
+       class_incref(target, __func__, current);
+
        if (target->obd_stopping || !target->obd_set_up) {
                spin_unlock(&target->obd_dev_lock);
 
        if (target->obd_stopping || !target->obd_set_up) {
                spin_unlock(&target->obd_dev_lock);
 
@@ -865,11 +977,6 @@ int target_handle_connect(struct ptlrpc_request *req)
                GOTO(out, rc = -EAGAIN);
        }
 
                GOTO(out, rc = -EAGAIN);
        }
 
-       /* Make sure the target isn't cleaned up while we're here. Yes,
-        * there's still a race between the above check and our incref here.
-        * Really, class_uuid2obd should take the ref. */
-       targref = class_incref(target, __FUNCTION__, current);
-
        target->obd_conn_inprogress++;
        spin_unlock(&target->obd_dev_lock);
 
        target->obd_conn_inprogress++;
        spin_unlock(&target->obd_dev_lock);
 
@@ -881,21 +988,6 @@ int target_handle_connect(struct ptlrpc_request *req)
 
         obd_str2uuid(&cluuid, str);
 
 
         obd_str2uuid(&cluuid, str);
 
-       /* XXX Extract a nettype and format accordingly. */
-       switch (sizeof(lnet_nid_t)) {
-       /* NB the casts only avoid compiler warnings. */
-        case 8:
-                snprintf(remote_uuid.uuid, sizeof remote_uuid,
-                         "NET_"LPX64"_UUID", (__u64)req->rq_peer.nid);
-                break;
-        case 4:
-                snprintf(remote_uuid.uuid, sizeof remote_uuid,
-                         "NET_%x_UUID", (__u32)req->rq_peer.nid);
-                break;
-        default:
-                LBUG();
-        }
-
         tmp = req_capsule_client_get(&req->rq_pill, &RMF_CONN);
         if (tmp == NULL)
                 GOTO(out, rc = -EPROTO);
         tmp = req_capsule_client_get(&req->rq_pill, &RMF_CONN);
         if (tmp == NULL)
                 GOTO(out, rc = -EPROTO);
@@ -1144,8 +1236,10 @@ dont_check_exports:
                        if (mds_conn && OBD_FAIL_CHECK(OBD_FAIL_TGT_RCVG_FLAG))
                                lustre_msg_add_op_flags(req->rq_repmsg,
                                                        MSG_CONNECT_RECOVERING);
                        if (mds_conn && OBD_FAIL_CHECK(OBD_FAIL_TGT_RCVG_FLAG))
                                lustre_msg_add_op_flags(req->rq_repmsg,
                                                        MSG_CONNECT_RECOVERING);
-                       if (rc == 0)
+                       if (rc == 0) {
                                conn.cookie = export->exp_handle.h_cookie;
                                conn.cookie = export->exp_handle.h_cookie;
+                               rc = rev_import_init(export);
+                       }
 
                        if (mds_mds_conn)
                                new_mds_mds_conn = true;
 
                        if (mds_mds_conn)
                                new_mds_mds_conn = true;
@@ -1173,11 +1267,6 @@ dont_check_exports:
                 memcpy(tmpdata, data, min(tmpsize, size));
         }
 
                 memcpy(tmpdata, data, min(tmpsize, size));
         }
 
-        /* If all else goes well, this is our RPC return code. */
-        req->rq_status = 0;
-
-        lustre_msg_set_handle(req->rq_repmsg, &conn);
-
         /* If the client and the server are the same node, we will already
          * have an export that really points to the client's DLM export,
          * because we have a shared handles table.
         /* If the client and the server are the same node, we will already
          * have an export that really points to the client's DLM export,
          * because we have a shared handles table.
@@ -1224,14 +1313,19 @@ dont_check_exports:
                 ptlrpc_connection_put(export->exp_connection);
         }
 
                 ptlrpc_connection_put(export->exp_connection);
         }
 
-        export->exp_connection = ptlrpc_connection_get(req->rq_peer,
-                                                       req->rq_self,
-                                                       &remote_uuid);
-       if (hlist_unhashed(&export->exp_nid_hash)) {
-                cfs_hash_add(export->exp_obd->obd_nid_hash,
-                             &export->exp_connection->c_peer.nid,
-                             &export->exp_nid_hash);
-        }
+       export->exp_connection = ptlrpc_connection_get(req->rq_peer,
+                                                      req->rq_self,
+                                                      &cluuid);
+       if (hlist_unhashed(&export->exp_nid_hash))
+               cfs_hash_add(export->exp_obd->obd_nid_hash,
+                            &export->exp_connection->c_peer.nid,
+                            &export->exp_nid_hash);
+
+       lustre_msg_set_handle(req->rq_repmsg, &conn);
+
+       rc = rev_import_reconnect(export, req);
+       if (rc != 0)
+               GOTO(out, rc);
 
        if (target->obd_recovering && !export->exp_in_recovery && !lw_client) {
                 int has_transno;
 
        if (target->obd_recovering && !export->exp_in_recovery && !lw_client) {
                 int has_transno;
@@ -1283,54 +1377,7 @@ dont_check_exports:
        if (target->obd_recovering && !lw_client)
                 lustre_msg_add_op_flags(req->rq_repmsg, MSG_CONNECT_RECOVERING);
 
        if (target->obd_recovering && !lw_client)
                 lustre_msg_add_op_flags(req->rq_repmsg, MSG_CONNECT_RECOVERING);
 
-        tmp = req_capsule_client_get(&req->rq_pill, &RMF_CONN);
-        conn = *tmp;
-
-       /* Return -ENOTCONN in case of errors to let client reconnect. */
-       revimp = class_new_import(target);
-       if (revimp == NULL) {
-               CERROR("fail to alloc new reverse import.\n");
-               GOTO(out, rc = -ENOTCONN);
-       }
-
-       spin_lock(&export->exp_lock);
-       if (export->exp_imp_reverse != NULL)
-               /* destroyed import can be still referenced in ctxt */
-               tmp_imp = export->exp_imp_reverse;
-       export->exp_imp_reverse = revimp;
-       spin_unlock(&export->exp_lock);
-
-        revimp->imp_connection = ptlrpc_connection_addref(export->exp_connection);
-        revimp->imp_client = &export->exp_obd->obd_ldlm_client;
-        revimp->imp_remote_handle = conn;
-        revimp->imp_dlm_fake = 1;
-        revimp->imp_state = LUSTRE_IMP_FULL;
-
-       /* Unknown versions will be caught in
-        * ptlrpc_handle_server_req_in->lustre_unpack_msg(). */
-        revimp->imp_msg_magic = req->rq_reqmsg->lm_magic;
-
-       if (data->ocd_connect_flags & OBD_CONNECT_AT)
-               revimp->imp_msghdr_flags |= MSGHDR_AT_SUPPORT;
-       else
-               revimp->imp_msghdr_flags &= ~MSGHDR_AT_SUPPORT;
-
-       revimp->imp_msghdr_flags |= MSGHDR_CKSUM_INCOMPAT18;
-
-       rc = sptlrpc_import_sec_adapt(revimp, req->rq_svc_ctx, &req->rq_flvr);
-       if (rc) {
-               CERROR("Failed to get sec for reverse import: %d\n", rc);
-               spin_lock(&export->exp_lock);
-               export->exp_imp_reverse = NULL;
-               spin_unlock(&export->exp_lock);
-               class_destroy_import(revimp);
-       }
-
-       class_import_put(revimp);
-
 out:
 out:
-       if (tmp_imp != NULL)
-               client_destroy_import(tmp_imp);
        if (export) {
                spin_lock(&export->exp_lock);
                export->exp_connecting = 0;
        if (export) {
                spin_lock(&export->exp_lock);
                export->exp_connecting = 0;
@@ -1338,15 +1385,14 @@ out:
 
                class_export_put(export);
        }
 
                class_export_put(export);
        }
-       if (targref) {
+       if (target != NULL) {
                spin_lock(&target->obd_dev_lock);
                target->obd_conn_inprogress--;
                spin_unlock(&target->obd_dev_lock);
 
                spin_lock(&target->obd_dev_lock);
                target->obd_conn_inprogress--;
                spin_unlock(&target->obd_dev_lock);
 
-               class_decref(targref, __func__, current);
+               class_decref(target, __func__, current);
        }
        }
-       if (rc)
-               req->rq_status = rc;
+       req->rq_status = rc;
        RETURN(rc);
 }
 
        RETURN(rc);
 }
 
index 8e816c5..4fb3ecc 100644 (file)
@@ -959,6 +959,7 @@ void class_unlink_export(struct obd_export *exp)
        /* A reference is kept by obd_stale_exports list */
        obd_stale_export_put(exp);
 }
        /* A reference is kept by obd_stale_exports list */
        obd_stale_export_put(exp);
 }
+EXPORT_SYMBOL(class_unlink_export);
 
 /* Import management functions */
 static void class_import_destroy(struct obd_import *imp)
 
 /* Import management functions */
 static void class_import_destroy(struct obd_import *imp)
index e8f595b..b663ca5 100644 (file)
@@ -103,6 +103,12 @@ do {                                                                       \
        spin_unlock(&imp->imp_lock);                                    \
 } while(0)
 
        spin_unlock(&imp->imp_lock);                                    \
 } while(0)
 
+void ptlrpc_import_enter_resend(struct obd_import *imp)
+{
+       IMPORT_SET_STATE(imp, LUSTRE_IMP_RECOVER);
+}
+EXPORT_SYMBOL(ptlrpc_import_enter_resend);
+
 
 static int ptlrpc_connect_interpret(const struct lu_env *env,
                                     struct ptlrpc_request *request,
 
 static int ptlrpc_connect_interpret(const struct lu_env *env,
                                     struct ptlrpc_request *request,
@@ -1473,9 +1479,7 @@ int ptlrpc_import_recovery_state_machine(struct obd_import *imp)
        }
 
         if (imp->imp_state == LUSTRE_IMP_RECOVER) {
        }
 
         if (imp->imp_state == LUSTRE_IMP_RECOVER) {
-                CDEBUG(D_HA, "reconnected to %s@%s\n",
-                       obd2cli_tgt(imp->imp_obd),
-                       imp->imp_connection->c_remote_uuid.uuid);
+               struct ptlrpc_connection *conn = imp->imp_connection;
 
                 rc = ptlrpc_resend(imp);
                 if (rc)
 
                 rc = ptlrpc_resend(imp);
                 if (rc)
@@ -1483,12 +1487,10 @@ int ptlrpc_import_recovery_state_machine(struct obd_import *imp)
                 IMPORT_SET_STATE(imp, LUSTRE_IMP_FULL);
                 ptlrpc_activate_import(imp);
 
                 IMPORT_SET_STATE(imp, LUSTRE_IMP_FULL);
                 ptlrpc_activate_import(imp);
 
-                deuuidify(obd2cli_tgt(imp->imp_obd), NULL,
-                          &target_start, &target_len);
-                LCONSOLE_INFO("%s: Connection restored to %.*s (at %s)\n",
-                              imp->imp_obd->obd_name,
-                              target_len, target_start,
-                              libcfs_nid2str(imp->imp_connection->c_peer.nid));
+               LCONSOLE_INFO("%s: Connection restored to %s (at %s)\n",
+                             imp->imp_obd->obd_name,
+                             obd_uuid2str(&conn->c_remote_uuid),
+                             libcfs_nid2str(imp->imp_connection->c_peer.nid));
         }
 
        if (imp->imp_state == LUSTRE_IMP_FULL) {
         }
 
        if (imp->imp_state == LUSTRE_IMP_FULL) {
index d1afc9d..8b0f9a0 100644 (file)
@@ -1459,6 +1459,12 @@ static int tgt_clients_data_init(const struct lu_env *env,
 
                class_export_put(exp);
 
 
                class_export_put(exp);
 
+               rc = rev_import_init(exp);
+               if (rc != 0) {
+                       class_unlink_export(exp);
+                       GOTO(err_out, rc);
+               }
+
                /* Need to check last_rcvd even for duplicated exports. */
                CDEBUG(D_OTHER, "client at idx %d has last_transno = "LPU64"\n",
                       cl_idx, last_transno);
                /* Need to check last_rcvd even for duplicated exports. */
                CDEBUG(D_OTHER, "client at idx %d has last_transno = "LPU64"\n",
                       cl_idx, last_transno);
index af5c209..8e75d0d 100755 (executable)
@@ -196,6 +196,53 @@ test_10b() {
 }
 run_test 10b "re-send BL AST"
 
 }
 run_test 10b "re-send BL AST"
 
+test_10c() {
+       local before=$(date +%s)
+       local evict
+       local mdccli
+       local mdcpath
+       local conn_uuid
+       local workdir
+       local pid
+       local rc
+
+       workdir="${DIR}/${tdir}"
+       mkdir -p ${workdir} || error "can't create workdir $?"
+       stat ${workdir} > /dev/null ||
+               error "failed to stat ${workdir}: $?"
+       mdtidx=$($LFS getdirstripe -i ${workdir})
+       mdtname=$($LFS mdts ${workdir} | grep -e "^$mdtidx:" |
+                 awk '{sub("_UUID", "", $2); print $2;}')
+       #assume one client
+       mdccli=$($LCTL dl | grep "${mdtname}-mdc" | awk '{print $4;}')
+       conn_uuid=$($LCTL get_param -n mdc.${mdccli}.mds_conn_uuid)
+       mdcpath="mdc.${mdccli}.import=connection=${conn_uuid}"
+
+       drop_bl_callback_once "chmod 0777 ${workdir}" &
+       pid=$!
+
+       # let chmod blocked
+       sleep 1
+       # force client reconnect
+       $LCTL set_param "${mdcpath}"
+
+       # wait client reconnect
+       client_reconnect
+       wait $pid
+       rc=$?
+       evict=$($LCTL get_param mdc.${mdccli}.state |
+          awk -F"[ [,]" '/EVICTED]$/ { if (t<$4) {t=$4;} } END { print t }')
+
+       [[ $evict -le $before ]] ||
+               ( $LCTL get_param mdc.$FSNAME-MDT*.state;
+                   error "eviction happened: $EVICT before:$BEFORE" )
+
+       [ $rc -eq 0 ] || error "chmod must finished OK"
+       checkstat -v -p 0777 "${workdir}" ||
+               error "client checkstat failed: $?"
+}
+run_test 10c "re-send BL AST vs reconnect race (LU-5569)"
+
 test_10d() {
        local before=$(date +%s)
        local evict
 test_10d() {
        local before=$(date +%s)
        local evict
@@ -231,6 +278,7 @@ test_10d() {
                (do_facet client $LCTL get_param osc.$FSNAME-OST0000*.state;
                    error "no eviction: $evict before:$before")
 
                (do_facet client $LCTL get_param osc.$FSNAME-OST0000*.state;
                    error "no eviction: $evict before:$before")
 
+       $LCTL set_param fail_err=0
        rm $TMP/$tfile
        umount_client $MOUNT2
 }
        rm $TMP/$tfile
        umount_client $MOUNT2
 }