LU-14594 ptlrpc: do not match reply with resent RPC

The server is able to filter by the connection ID, and drop late
coming RPCs of previous connections, however it does not happen for
replies. At the same time, this is a problem in some cases.

Allocate new matchbits for resends and check replies by them, instead
of xid. Connect RPCs are exceptions due to interop with old server -
at the time of connect we do not know yet if the server supports it.

Signed-off-by: Vitaly Fertman <c17818@cray.com>
Change-Id: I2aad037002b488b0c3371544ede0c47940f87efe
HPE-bug-id: LUS-9596
Reviewed-on: https://es-gerrit.dev.cray.com/158446
Reviewed-by: Alexey Lyashkov <c17817@cray.com>
Reviewed-by: Andriy Skulysh <c17819@cray.com>
Tested-by: Elena Gryaznova <c17455@cray.com>
Reviewed-on: https://review.whamcloud.com/43242
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Mike Pershin <mpershin@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/include/lustre_net.h b/lustre/include/lustre_net.h
index 466e484..843061e 100644 (file)
--- a/lustre/include/lustre_net.h
+++ b/lustre/include/lustre_net.h
@@ -1039,6 +1039,8 @@ struct ptlrpc_request {
         __u64                           rq_xid;
        /** bulk match bits */
        __u64                            rq_mbits;
+       /** reply match bits */
+       __u64                            rq_rep_mbits;
        /**
         * List item to for replay list. Not yet committed requests get linked
         * there.

diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h
index 4b15d6a..d24d3f1 100644 (file)
--- a/lustre/include/obd_support.h
+++ b/lustre/include/obd_support.h
@@ -457,6 +457,7 @@ extern char obd_jobid_var[];
 #define OBD_FAIL_PTLRPC_CONNECT_RACE    0x531
 #define OBD_FAIL_NET_ERROR_RPC          0x532
 #define OBD_FAIL_PTLRPC_IDLE_RACE       0x533
+#define OBD_FAIL_PTLRPC_ENQ_RESEND      0x534
 
 #define OBD_FAIL_OBD_PING_NET            0x600
 /*     OBD_FAIL_OBD_LOG_CANCEL_NET      0x601 obsolete since 1.5 */

diff --git a/lustre/include/uapi/linux/lustre/lustre_idl.h b/lustre/include/uapi/linux/lustre/lustre_idl.h
index b0f22cb..de854ee 100644 (file)
--- a/lustre/include/uapi/linux/lustre/lustre_idl.h
+++ b/lustre/include/uapi/linux/lustre/lustre_idl.h
@@ -850,6 +850,7 @@ struct ptlrpc_body_v2 {
 #define OBD_CONNECT2_GETATTR_PFID      0x20000ULL /* pack parent FID in getattr */
 #define OBD_CONNECT2_LSEEK            0x40000ULL /* SEEK_HOLE/DATA RPC */
 #define OBD_CONNECT2_DOM_LVB          0x80000ULL /* pack DOM glimpse data in LVB */
+#define OBD_CONNECT2_REP_MBITS         0x100000ULL /* match reply by mbits, not xid */
 /* XXX README XXX:
  * Please DO NOT add flag values here before first ensuring that this same
  * flag value is not in use on some other branch.  Please clear any such
@@ -909,7 +910,8 @@ struct ptlrpc_body_v2 {
                                OBD_CONNECT2_CRUSH | \
                                OBD_CONNECT2_ENCRYPT | \
                                OBD_CONNECT2_GETATTR_PFID |\
-                               OBD_CONNECT2_LSEEK | OBD_CONNECT2_DOM_LVB)
+                               OBD_CONNECT2_LSEEK | OBD_CONNECT2_DOM_LVB |\
+                               OBD_CONNECT2_REP_MBITS)
 
 #define OST_CONNECT_SUPPORTED  (OBD_CONNECT_SRVLOCK | OBD_CONNECT_GRANT | \
                                OBD_CONNECT_REQPORTAL | OBD_CONNECT_VERSION | \
@@ -931,17 +933,19 @@ struct ptlrpc_body_v2 {
                                OBD_CONNECT_SHORTIO | OBD_CONNECT_FLAGS2)
 
 #define OST_CONNECT_SUPPORTED2 (OBD_CONNECT2_LOCKAHEAD | OBD_CONNECT2_INC_XID |\
-                               OBD_CONNECT2_ENCRYPT | OBD_CONNECT2_LSEEK)
+                               OBD_CONNECT2_ENCRYPT | OBD_CONNECT2_LSEEK |\
+                               OBD_CONNECT2_REP_MBITS)
 
-#define ECHO_CONNECT_SUPPORTED (OBD_CONNECT_FID)
-#define ECHO_CONNECT_SUPPORTED2 0
+#define ECHO_CONNECT_SUPPORTED (OBD_CONNECT_FID | OBD_CONNECT_FLAGS2)
+#define ECHO_CONNECT_SUPPORTED2 OBD_CONNECT2_REP_MBITS
 
 #define MGS_CONNECT_SUPPORTED  (OBD_CONNECT_VERSION | OBD_CONNECT_AT | \
                                OBD_CONNECT_FULL20 | OBD_CONNECT_IMP_RECOV | \
                                OBD_CONNECT_PINGLESS |\
-                               OBD_CONNECT_BULK_MBITS | OBD_CONNECT_BARRIER)
+                               OBD_CONNECT_BULK_MBITS | OBD_CONNECT_BARRIER | \
+                               OBD_CONNECT_FLAGS2)
 
-#define MGS_CONNECT_SUPPORTED2 0
+#define MGS_CONNECT_SUPPORTED2 OBD_CONNECT2_REP_MBITS
 
 /* Features required for this version of the client to work with server */
 #define CLIENT_CONNECT_MDT_REQD (OBD_CONNECT_FID |     \

diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c
index efc80c1..bb6a2f0 100644 (file)
--- a/lustre/ldlm/ldlm_lib.c
+++ b/lustre/ldlm/ldlm_lib.c
@@ -982,6 +982,7 @@ static int rev_import_flags_update(struct obd_import *revimp,
 
        revimp->imp_msghdr_flags |= MSGHDR_CKSUM_INCOMPAT18;
 
+       revimp->imp_connect_data = *data;
        rc = sptlrpc_import_sec_adapt(revimp, req->rq_svc_ctx, &req->rq_flvr);
        if (rc) {
                CERROR("%s: cannot get reverse import %s security: rc = %d\n",

diff --git a/lustre/llite/llite_lib.c b/lustre/llite/llite_lib.c
index 62a3faa..0388491 100644 (file)
--- a/lustre/llite/llite_lib.c
+++ b/lustre/llite/llite_lib.c
@@ -335,7 +335,8 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt)
                                   OBD_CONNECT2_PCC |
                                   OBD_CONNECT2_CRUSH | OBD_CONNECT2_LSEEK |
                                   OBD_CONNECT2_GETATTR_PFID |
-                                  OBD_CONNECT2_DOM_LVB;
+                                  OBD_CONNECT2_DOM_LVB |
+                                  OBD_CONNECT2_REP_MBITS;
 
 #ifdef HAVE_LRU_RESIZE_SUPPORT
         if (sbi->ll_flags & LL_SBI_LRU_RESIZE)
@@ -527,7 +528,8 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt)
                                  OBD_CONNECT_BULK_MBITS | OBD_CONNECT_SHORTIO |
                                  OBD_CONNECT_FLAGS2 | OBD_CONNECT_GRANT_SHRINK;
        data->ocd_connect_flags2 = OBD_CONNECT2_LOCKAHEAD |
-                                  OBD_CONNECT2_INC_XID | OBD_CONNECT2_LSEEK;
+                                  OBD_CONNECT2_INC_XID | OBD_CONNECT2_LSEEK |
+                                  OBD_CONNECT2_REP_MBITS;
 
        if (!OBD_FAIL_CHECK(OBD_FAIL_OSC_CONNECT_GRANT_PARAM))
                data->ocd_connect_flags |= OBD_CONNECT_GRANT_PARAM;

diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c
index aae5616..a74ce0b 100644 (file)
--- a/lustre/mdt/mdt_handler.c
+++ b/lustre/mdt/mdt_handler.c
@@ -2130,6 +2130,16 @@ static int mdt_getattr_name_lock(struct mdt_thread_info *info,
                         PLDLMRES(lock->l_resource),
                         PFID(mdt_object_fid(child)));
 
+               if (unlikely(OBD_FAIL_PRECHECK(OBD_FAIL_PTLRPC_ENQ_RESEND))) {
+                       if (!(lustre_msg_get_flags(req->rq_reqmsg) & MSG_RESENT))
+                               OBD_FAIL_TIMEOUT(OBD_FAIL_PTLRPC_ENQ_RESEND,
+                                                req->rq_deadline -
+                                                req->rq_arrival_time.tv_sec +
+                                                cfs_fail_val ?: 3);
+                       /* Put the lock to the waiting list and force the cancel */
+                       ldlm_set_ast_sent(lock);
+               }
+
                if (S_ISREG(lu_object_attr(&child->mot_obj)) &&
                    !mdt_object_remote(child) && child != parent) {
                        mdt_object_put(info->mti_env, child);

diff --git a/lustre/mdt/mdt_reint.c b/lustre/mdt/mdt_reint.c
index e6464a7..adf02cc 100644 (file)
--- a/lustre/mdt/mdt_reint.c
+++ b/lustre/mdt/mdt_reint.c
@@ -1295,7 +1295,8 @@ static int mdt_reint_link(struct mdt_thread_info *info,
        if (OBD_FAIL_CHECK(OBD_FAIL_MDS_REINT_LINK))
                RETURN(err_serious(-ENOENT));
 
-       if (OBD_FAIL_PRECHECK(OBD_FAIL_PTLRPC_RESEND_RACE)) {
+       if (OBD_FAIL_PRECHECK(OBD_FAIL_PTLRPC_RESEND_RACE) ||
+           OBD_FAIL_PRECHECK(OBD_FAIL_PTLRPC_ENQ_RESEND)) {
                req->rq_no_reply = 1;
                RETURN(err_serious(-ENOENT));
        }

diff --git a/lustre/obdclass/lprocfs_status.c b/lustre/obdclass/lprocfs_status.c
index 5640acb..ac0d0aa 100644 (file)
--- a/lustre/obdclass/lprocfs_status.c
+++ b/lustre/obdclass/lprocfs_status.c
@@ -641,6 +641,7 @@ static const char *const obd_connect_names[] = {
        "getattr_pfid",         /* 0x20000 */
        "lseek",                /* 0x40000 */
        "dom_lvb",              /* 0x80000 */
+       "reply_mbits",          /* 0x100000 */
        NULL
 };
 

diff --git a/lustre/obdclass/obd_mount.c b/lustre/obdclass/obd_mount.c
index 56fc2e3..53b359c 100644 (file)
--- a/lustre/obdclass/obd_mount.c
+++ b/lustre/obdclass/obd_mount.c
@@ -461,7 +461,9 @@ int lustre_start_mgc(struct super_block *sb)
        data->ocd_connect_flags = OBD_CONNECT_VERSION | OBD_CONNECT_AT |
                                  OBD_CONNECT_FULL20 | OBD_CONNECT_IMP_RECOV |
                                  OBD_CONNECT_LVB_TYPE |
-                                 OBD_CONNECT_BULK_MBITS | OBD_CONNECT_BARRIER;
+                                 OBD_CONNECT_BULK_MBITS | OBD_CONNECT_BARRIER |
+                                 OBD_CONNECT_FLAGS2;
+       data->ocd_connect_flags2 = OBD_CONNECT2_REP_MBITS;
 
        if (lmd_is_client(lsi->lsi_lmd) &&
            lsi->lsi_lmd->lmd_flags & LMD_FLG_NOIR)

diff --git a/lustre/obdecho/echo_client.c b/lustre/obdecho/echo_client.c
index c10f532..437466a 100644 (file)
--- a/lustre/obdecho/echo_client.c
+++ b/lustre/obdecho/echo_client.c
@@ -3009,7 +3009,9 @@ static int echo_client_setup(const struct lu_env *env,
                                 OBD_CONNECT_BRW_SIZE |
                                 OBD_CONNECT_GRANT | OBD_CONNECT_FULL20 |
                                 OBD_CONNECT_64BITHASH | OBD_CONNECT_LVB_TYPE |
-                                OBD_CONNECT_FID;
+                                OBD_CONNECT_FID | OBD_CONNECT_FLAGS2;
+       ocd->ocd_connect_flags2 = OBD_CONNECT2_REP_MBITS;
+
        ocd->ocd_brw_size = DT_MAX_BRW_SIZE;
        ocd->ocd_version = LUSTRE_VERSION_CODE;
        ocd->ocd_group = FID_SEQ_ECHO;

diff --git a/lustre/ptlrpc/client.c b/lustre/ptlrpc/client.c
index 8c23597..67ee2b4 100644 (file)
--- a/lustre/ptlrpc/client.c
+++ b/lustre/ptlrpc/client.c
@@ -3421,12 +3421,11 @@ __u64 ptlrpc_next_xid(void)
  * request to ensure previous bulk fails and avoid problems with lost replies
  * and therefore several transfers landing into the same buffer from different
  * sending attempts.
+ * Also, to avoid previous reply landing to a different sending attempt.
  */
-void ptlrpc_set_bulk_mbits(struct ptlrpc_request *req)
+void ptlrpc_set_mbits(struct ptlrpc_request *req)
 {
-       struct ptlrpc_bulk_desc *bd = req->rq_bulk;
-
-       LASSERT(bd != NULL);
+       int md_count = req->rq_bulk ? req->rq_bulk->bd_md_count : 1;
 
        /*
         * Generate new matchbits for all resend requests, including
@@ -3442,7 +3441,7 @@ void ptlrpc_set_bulk_mbits(struct ptlrpc_request *req)
                 * 'resend for the -EINPROGRESS resend'. To make it simple,
                 * we opt to generate mbits for all resend cases.
                 */
-               if (OCD_HAS_FLAG(&bd->bd_import->imp_connect_data,
+               if (OCD_HAS_FLAG(&req->rq_import->imp_connect_data,
                                 BULK_MBITS)) {
                        req->rq_mbits = ptlrpc_next_xid();
                } else {
@@ -3456,15 +3455,16 @@ void ptlrpc_set_bulk_mbits(struct ptlrpc_request *req)
                        spin_unlock(&req->rq_import->imp_lock);
                        req->rq_mbits = req->rq_xid;
                }
-               CDEBUG(D_HA, "resend bulk old x%llu new x%llu\n",
+               CDEBUG(D_HA, "resend with new mbits old x%llu new x%llu\n",
                       old_mbits, req->rq_mbits);
        } else if (!(lustre_msg_get_flags(req->rq_reqmsg) & MSG_REPLAY)) {
                /* Request being sent first time, use xid as matchbits. */
-               if (OCD_HAS_FLAG(&bd->bd_import->imp_connect_data, BULK_MBITS)
-                   || req->rq_mbits == 0) {
+               if (OCD_HAS_FLAG(&req->rq_import->imp_connect_data,
+                                BULK_MBITS) || req->rq_mbits == 0)
+               {
                        req->rq_mbits = req->rq_xid;
                } else {
-                       req->rq_mbits -= bd->bd_md_count - 1;
+                       req->rq_mbits -= md_count - 1;
                }
        } else {
                /*
@@ -3479,7 +3479,7 @@ void ptlrpc_set_bulk_mbits(struct ptlrpc_request *req)
         * that server can infer the number of bulks that were prepared,
         * see LU-1431
         */
-       req->rq_mbits += bd->bd_md_count - 1;
+       req->rq_mbits += md_count - 1;
 
        /*
         * Set rq_xid as rq_mbits to indicate the final bulk for the old
@@ -3488,7 +3488,7 @@ void ptlrpc_set_bulk_mbits(struct ptlrpc_request *req)
         * It's ok to directly set the rq_xid here, since this xid bump
         * won't affect the request position in unreplied list.
         */
-       if (!OCD_HAS_FLAG(&bd->bd_import->imp_connect_data, BULK_MBITS))
+       if (!OCD_HAS_FLAG(&req->rq_import->imp_connect_data, BULK_MBITS))
                req->rq_xid = req->rq_mbits;
 }
 

diff --git a/lustre/ptlrpc/niobuf.c b/lustre/ptlrpc/niobuf.c
index bf70c2b..eea686e 100644 (file)
--- a/lustre/ptlrpc/niobuf.c
+++ b/lustre/ptlrpc/niobuf.c
@@ -644,7 +644,8 @@ int ptlrpc_send_reply(struct ptlrpc_request *req, int flags)
                          LNET_ACK_REQ : LNET_NOACK_REQ,
                          &rs->rs_cb_id, req->rq_self, req->rq_source,
                          ptlrpc_req2svc(req)->srv_rep_portal,
-                         req->rq_xid, req->rq_reply_off, NULL);
+                         req->rq_rep_mbits ? req->rq_rep_mbits : req->rq_xid,
+                         req->rq_reply_off, NULL);
 out:
         if (unlikely(rc != 0))
                 ptlrpc_req_drop_rs(req);
@@ -701,7 +702,9 @@ int ptlrpc_error(struct ptlrpc_request *req)
 int ptl_send_rpc(struct ptlrpc_request *request, int noreply)
 {
        int rc;
+       __u32 opc;
        int mpflag = 0;
+       bool rep_mbits = false;
        struct lnet_handle_md bulk_cookie;
        struct ptlrpc_connection *connection;
        struct lnet_me *reply_me = NULL;
@@ -769,8 +772,14 @@ int ptl_send_rpc(struct ptlrpc_request *request, int noreply)
                          "Allocating new XID for resend on EINPROGRESS");
        }
 
-       if (request->rq_bulk != NULL) {
-               ptlrpc_set_bulk_mbits(request);
+       opc = lustre_msg_get_opc(request->rq_reqmsg);
+       if (opc != OST_CONNECT && opc != MDS_CONNECT &&
+           opc != MGS_CONNECT && OCD_HAS_FLAG(&imp->imp_connect_data, FLAGS2))
+               rep_mbits = imp->imp_connect_data.ocd_connect_flags2 &
+                       OBD_CONNECT2_REP_MBITS;
+
+       if ((request->rq_bulk != NULL) || rep_mbits) {
+               ptlrpc_set_mbits(request);
                lustre_msg_set_mbits(request->rq_reqmsg, request->rq_mbits);
        }
 
@@ -842,8 +851,9 @@ int ptl_send_rpc(struct ptlrpc_request *request, int noreply)
                } else {
                        reply_me = LNetMEAttach(request->rq_reply_portal,
                                                connection->c_peer,
-                                               request->rq_xid, 0,
-                                               LNET_UNLINK, LNET_INS_AFTER);
+                                               rep_mbits ? request->rq_mbits :
+                                               request->rq_xid,
+                                               0, LNET_UNLINK, LNET_INS_AFTER);
                }
 
                if (IS_ERR(reply_me)) {

diff --git a/lustre/ptlrpc/ptlrpc_internal.h b/lustre/ptlrpc/ptlrpc_internal.h
index eb26370..8c38908 100644 (file)
--- a/lustre/ptlrpc/ptlrpc_internal.h
+++ b/lustre/ptlrpc/ptlrpc_internal.h
@@ -87,7 +87,7 @@ void ptlrpc_set_add_new_req(struct ptlrpcd_ctl *pc,
 void ptlrpc_expired_set(struct ptlrpc_request_set *set);
 time64_t ptlrpc_set_next_timeout(struct ptlrpc_request_set *);
 void ptlrpc_resend_req(struct ptlrpc_request *request);
-void ptlrpc_set_bulk_mbits(struct ptlrpc_request *req);
+void ptlrpc_set_mbits(struct ptlrpc_request *req);
 void ptlrpc_assign_next_xid_nolock(struct ptlrpc_request *req);
 __u64 ptlrpc_known_replied_xid(struct obd_import *imp);
 void ptlrpc_add_unreplied(struct ptlrpc_request *req);

diff --git a/lustre/ptlrpc/service.c b/lustre/ptlrpc/service.c
index bbe46a8..9bc2bcd 100644 (file)
--- a/lustre/ptlrpc/service.c
+++ b/lustre/ptlrpc/service.c
@@ -1352,7 +1352,8 @@ static int ptlrpc_at_send_early_reply(struct ptlrpc_request *req)
 
        ENTRY;
 
-       if (CFS_FAIL_CHECK(OBD_FAIL_TGT_REPLAY_RECONNECT)) {
+       if (CFS_FAIL_CHECK(OBD_FAIL_TGT_REPLAY_RECONNECT) ||
+           CFS_FAIL_PRECHECK(OBD_FAIL_PTLRPC_ENQ_RESEND)) {
                /* don't send early reply */
                RETURN(1);
        }
@@ -1858,6 +1859,7 @@ static int ptlrpc_server_request_add(struct ptlrpc_service_part *svcpt,
                                ptlrpc_at_remove_timed(orig);
                        spin_unlock(&orig->rq_rqbd->rqbd_svcpt->scp_at_lock);
                        orig->rq_deadline = req->rq_deadline;
+                       orig->rq_rep_mbits = req->rq_rep_mbits;
                        if (likely(linked))
                                ptlrpc_at_add_timed(orig);
                        ptlrpc_server_drop_request(orig);
@@ -2051,6 +2053,7 @@ static int ptlrpc_server_handle_req_in(struct ptlrpc_service_part *svcpt,
        struct ptlrpc_service *svc = svcpt->scp_service;
        struct ptlrpc_request *req;
        __u32 deadline;
+       __u32 opc;
        int rc;
 
        ENTRY;
@@ -2107,8 +2110,9 @@ static int ptlrpc_server_handle_req_in(struct ptlrpc_service_part *svcpt,
                goto err_req;
        }
 
+       opc = lustre_msg_get_opc(req->rq_reqmsg);
        if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_DROP_REQ_OPC) &&
-           lustre_msg_get_opc(req->rq_reqmsg) == cfs_fail_val) {
+           opc == cfs_fail_val) {
                CERROR("drop incoming rpc opc %u, x%llu\n",
                       cfs_fail_val, req->rq_xid);
                goto err_req;
@@ -2122,7 +2126,7 @@ static int ptlrpc_server_handle_req_in(struct ptlrpc_service_part *svcpt,
                goto err_req;
        }
 
-       switch (lustre_msg_get_opc(req->rq_reqmsg)) {
+       switch (opc) {
        case MDS_WRITEPAGE:
        case OST_WRITE:
        case OUT_UPDATE:
@@ -2193,8 +2197,20 @@ static int ptlrpc_server_handle_req_in(struct ptlrpc_service_part *svcpt,
                thread->t_env->le_ses = &req->rq_session;
        }
 
+
+       if (unlikely(OBD_FAIL_PRECHECK(OBD_FAIL_PTLRPC_ENQ_RESEND) &&
+                    (opc == LDLM_ENQUEUE) &&
+                    (lustre_msg_get_flags(req->rq_reqmsg) & MSG_RESENT)))
+               OBD_FAIL_TIMEOUT(OBD_FAIL_PTLRPC_ENQ_RESEND, 6);
+
        ptlrpc_at_add_timed(req);
 
+       if (opc != OST_CONNECT && opc != MDS_CONNECT &&
+           opc != MGS_CONNECT && req->rq_export != NULL) {
+               if (exp_connect_flags2(req->rq_export) & OBD_CONNECT2_REP_MBITS)
+                       req->rq_rep_mbits = lustre_msg_get_mbits(req->rq_reqmsg);
+       }
+
        /* Move it over to the request processing queue */
        rc = ptlrpc_server_request_add(svcpt, req);
        if (rc)

diff --git a/lustre/ptlrpc/wiretest.c b/lustre/ptlrpc/wiretest.c
index c20035b..a3fb406 100644 (file)
--- a/lustre/ptlrpc/wiretest.c
+++ b/lustre/ptlrpc/wiretest.c
@@ -1382,6 +1382,8 @@ void lustre_assert_wire_constants(void)
                 OBD_CONNECT2_LSEEK);
        LASSERTF(OBD_CONNECT2_DOM_LVB == 0x80000ULL, "found 0x%.16llxULL\n",
                 OBD_CONNECT2_DOM_LVB);
+       LASSERTF(OBD_CONNECT2_REP_MBITS == 0x100000ULL, "found 0x%.16llxULL\n",
+                OBD_CONNECT2_REP_MBITS);
        LASSERTF(OBD_CKSUM_CRC32 == 0x00000001UL, "found 0x%.8xUL\n",
                (unsigned)OBD_CKSUM_CRC32);
        LASSERTF(OBD_CKSUM_ADLER == 0x00000002UL, "found 0x%.8xUL\n",

diff --git a/lustre/tests/sanityn.sh b/lustre/tests/sanityn.sh
index 1dd0838..860a4a9 100755 (executable)
--- a/lustre/tests/sanityn.sh
+++ b/lustre/tests/sanityn.sh
@@ -5606,6 +5606,48 @@ test_109() {
 
 run_test 109 "Race with several mount instances on 1 node"
 
+test_110() {
+       local before=$(date +%s)
+       local evict
+
+       mkdir -p $DIR/$tdir
+       touch $DIR/$tdir/f1
+       touch $DIR/$tfile
+
+       #define OBD_FAIL_PTLRPC_RESEND_RACE      0x525
+       do_facet mds1 lctl set_param fail_loc=0x525 fail_val=3
+
+       # disable last_xid logic by dropping link reply
+       ln $DIR/$tdir/f1 $DIR/$tdir/f2 &
+       sleep 1
+
+       #define OBD_FAIL_PTLRPC_ENQ_RESEND      0x534
+       do_facet mds1 lctl set_param fail_loc=0x534
+
+       # RPC will race with its Resend and the Resend will sleep to let
+       # the original lock to get granted & cancelled.
+       #
+       # AST_SENT is set artificially, so an explicit conflict is not needed
+       #
+       # The woken up Resend gets a new lock, but client does not wait for it
+       stat $DIR/$tfile
+       sleep $TIMEOUT
+       do_facet mds1 lctl set_param fail_loc=0 fail_val=0
+
+       # Take a conflict to wait long enough to see the eviction
+       touch $DIR2/$tfile
+
+       # let the client reconnect
+       client_reconnect
+       evict=$(do_facet client $LCTL get_param mdc.$FSNAME-MDT*.state |
+         awk -F"[ [,]" '/EVICTED ]$/ { if (mx<$5) {mx=$5;} } END { print mx }')
+
+       [ -z "$evict" ] || [[ $evict -le $before ]] ||
+               (do_facet client $LCTL get_param mdc.$FSNAME-MDT*.state;
+                   error "eviction happened: $evict before:$before")
+}
+run_test 110 "do not grant another lock on resend"
+
 log "cleanup: ======================================================"
 
 # kill and wait in each test only guarentee script finish, but command in script

diff --git a/lustre/utils/wirecheck.c b/lustre/utils/wirecheck.c
index f894e93..a7f5c66 100644 (file)
--- a/lustre/utils/wirecheck.c
+++ b/lustre/utils/wirecheck.c
@@ -627,6 +627,7 @@ check_obd_connect_data(void)
        CHECK_DEFINE_64X(OBD_CONNECT2_GETATTR_PFID);
        CHECK_DEFINE_64X(OBD_CONNECT2_LSEEK);
        CHECK_DEFINE_64X(OBD_CONNECT2_DOM_LVB);
+       CHECK_DEFINE_64X(OBD_CONNECT2_REP_MBITS);
 
        CHECK_VALUE_X(OBD_CKSUM_CRC32);
        CHECK_VALUE_X(OBD_CKSUM_ADLER);

diff --git a/lustre/utils/wiretest.c b/lustre/utils/wiretest.c
index 3b3f961..9f2b869 100644 (file)
--- a/lustre/utils/wiretest.c
+++ b/lustre/utils/wiretest.c
@@ -1404,6 +1404,8 @@ void lustre_assert_wire_constants(void)
                 OBD_CONNECT2_LSEEK);
        LASSERTF(OBD_CONNECT2_DOM_LVB == 0x80000ULL, "found 0x%.16llxULL\n",
                 OBD_CONNECT2_DOM_LVB);
+       LASSERTF(OBD_CONNECT2_REP_MBITS == 0x100000ULL, "found 0x%.16llxULL\n",
+                OBD_CONNECT2_REP_MBITS);
        LASSERTF(OBD_CKSUM_CRC32 == 0x00000001UL, "found 0x%.8xUL\n",
                (unsigned)OBD_CKSUM_CRC32);
        LASSERTF(OBD_CKSUM_ADLER == 0x00000002UL, "found 0x%.8xUL\n",