branch: b1_6

do not drop replay according to msg flags, instead we check the
per-export recovery request queue for duplication of transno.
b=15756
r=adilger
r=rread

diff --git a/lustre/ChangeLog b/lustre/ChangeLog
index c83935f..865ce02 100644 (file)
--- a/lustre/ChangeLog
+++ b/lustre/ChangeLog
@@ -25,6 +25,13 @@ tbd Sun Microsystems, Inc.
          For more information, please refer to bugzilla 13904.
 
 Severity   : normal
+Bugzilla   : 15756
+Frequency  : rare, replay get lost on server
+Description: server incorrectly drop resent replays lead to recovery failure.
+Details    : do not drop replay according to msg flags, instead we check the
+             per-export recovery request queue for duplication of transno.
+
+Severity   : normal
 Bugzilla   : 14835
 Frequency  : after recovery
 Description: precreate to many object's after del orphan.

diff --git a/lustre/include/lustre_export.h b/lustre/include/lustre_export.h
index 7ece7a2..6890dbd 100644 (file)
--- a/lustre/include/lustre_export.h
+++ b/lustre/include/lustre_export.h
@@ -88,6 +88,7 @@ struct obd_export {
         struct ldlm_export_data   exp_ldlm_data;
         struct list_head          exp_outstanding_replies;
         time_t                    exp_last_request_time;
+        struct list_head          exp_req_replay_queue;
         spinlock_t                exp_lock; /* protects flags int below */
         /* ^ protects exp_outstanding_replies too */
         __u64                     exp_connect_flags;

diff --git a/lustre/include/obd_support.h b/lustre/include/obd_support.h
index 792a6d4..2ee8438 100644 (file)
--- a/lustre/include/obd_support.h
+++ b/lustre/include/obd_support.h
@@ -254,6 +254,7 @@ extern unsigned int obd_alloc_fail_rate;
 #define OBD_FAIL_TGT_DELAY_RECONNECT     0x704
 #define OBD_FAIL_TGT_DELAY_PRECREATE     0x705
 #define OBD_FAIL_TGT_TOOMANY_THREADS     0x706
+#define OBD_FAIL_TGT_REPLAY_DROP         0x707
 
 #define OBD_FAIL_MDC_REVALIDATE_PAUSE    0x800
 #define OBD_FAIL_MDC_ENQUEUE_PAUSE       0x801

diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c
index 50e769f..cb5c4ef 100644 (file)
--- a/lustre/ldlm/ldlm_lib.c
+++ b/lustre/ldlm/ldlm_lib.c
@@ -973,6 +973,47 @@ void target_destroy_export(struct obd_export *exp)
  * Recovery functions
  */
 
+static int target_exp_enqueue_req_replay(struct ptlrpc_request *req)
+{
+        __u64                  transno = lustre_msg_get_transno(req->rq_reqmsg);
+        struct obd_export     *exp = req->rq_export;
+        struct ptlrpc_request *reqiter;
+        int                    dup = 0;
+
+        LASSERT(exp);
+
+        spin_lock(&exp->exp_lock);
+        list_for_each_entry(reqiter, &exp->exp_req_replay_queue,
+                            rq_replay_list) {
+                if (lustre_msg_get_transno(reqiter->rq_reqmsg) == transno) {
+                        dup = 1;
+                        break;
+                }
+        }
+
+        if (dup) {
+                /* we expect it with RESENT and REPLAY flags */
+                if ((lustre_msg_get_flags(req->rq_reqmsg) &
+                     (MSG_RESENT | MSG_REPLAY)) != (MSG_RESENT | MSG_REPLAY))
+                        CERROR("invalid flags %x of resent replay\n",
+                               lustre_msg_get_flags(req->rq_reqmsg));
+        } else {
+                list_add_tail(&req->rq_replay_list, &exp->exp_req_replay_queue);
+        }
+
+        spin_unlock(&exp->exp_lock);
+        return dup;
+}
+
+static void target_exp_dequeue_req_replay(struct ptlrpc_request *req)
+{
+        LASSERT(!list_empty(&req->rq_replay_list));
+        LASSERT(req->rq_export);
+
+        spin_lock(&req->rq_export->exp_lock);
+        list_del_init(&req->rq_replay_list);
+        spin_unlock(&req->rq_export->exp_lock);
+}
 
 static void target_release_saved_req(struct ptlrpc_request *req)
 {
@@ -1017,6 +1058,7 @@ static void abort_recovery_queue(struct obd_device *obd)
 
         list_for_each_safe(tmp, n, &obd->obd_recovery_queue) {
                 req = list_entry(tmp, struct ptlrpc_request, rq_list);
+                target_exp_dequeue_req_replay(req);
                 list_del(&req->rq_list);
                 DEBUG_REQ(D_ERROR, req, "aborted:");
                 req->rq_status = -ENOTCONN;
@@ -1066,6 +1108,7 @@ void target_cleanup_recovery(struct obd_device *obd)
 
         list_for_each_safe(tmp, n, &obd->obd_recovery_queue) {
                 req = list_entry(tmp, struct ptlrpc_request, rq_list);
+                target_exp_dequeue_req_replay(req);
                 list_del(&req->rq_list);
                 target_release_saved_req(req);
         }
@@ -1278,6 +1321,7 @@ static void process_recovery_queue(struct obd_device *obd)
                         }
                         continue;
                 }
+                target_exp_dequeue_req_replay(req);
                 list_del_init(&req->rq_list);
                 obd->obd_requests_queued_for_recovery--;
                 spin_unlock_bh(&obd->obd_processing_task_lock);
@@ -1314,6 +1358,7 @@ int target_queue_recovery_request(struct ptlrpc_request *req,
         __u64 transno = lustre_msg_get_transno(req->rq_reqmsg);
         struct ptlrpc_request *saved_req;
         struct lustre_msg *reqmsg;
+        int rc = 0;
 
         /* CAVEAT EMPTOR: The incoming request message has been swabbed
          * (i.e. buflens etc are in my own byte order), but type-dependent
@@ -1351,20 +1396,12 @@ int target_queue_recovery_request(struct ptlrpc_request *req,
                 /* Processing the queue right now, don't re-add. */
                 LASSERT(list_empty(&req->rq_list));
                 spin_unlock_bh(&obd->obd_processing_task_lock);
-                OBD_FREE(reqmsg, req->rq_reqlen);
-                OBD_FREE(saved_req, sizeof *saved_req);
-                return 1;
+                GOTO(err_free, rc = 1);
         }
 
-        /* A resent, replayed request that is still on the queue; just drop it.
-           The queued request will handle this. */
-        if ((lustre_msg_get_flags(req->rq_reqmsg) & (MSG_RESENT|MSG_REPLAY)) ==
-            (MSG_RESENT | MSG_REPLAY)) {
-                DEBUG_REQ(D_ERROR, req, "dropping resent queued req");
+        if (unlikely(OBD_FAIL_CHECK(OBD_FAIL_TGT_REPLAY_DROP))) {
                 spin_unlock_bh(&obd->obd_processing_task_lock);
-                OBD_FREE(reqmsg, req->rq_reqlen);
-                OBD_FREE(saved_req, sizeof *saved_req);
-                return 0;
+                GOTO(err_free, rc = 0);
         }
 
         memcpy(saved_req, req, sizeof *req);
@@ -1373,6 +1410,13 @@ int target_queue_recovery_request(struct ptlrpc_request *req,
         req->rq_reqmsg = reqmsg;
         class_export_get(req->rq_export);
         CFS_INIT_LIST_HEAD(&req->rq_list);
+        CFS_INIT_LIST_HEAD(&req->rq_replay_list);
+
+        if (target_exp_enqueue_req_replay(req)) {
+                spin_unlock_bh(&obd->obd_processing_task_lock);
+                DEBUG_REQ(D_ERROR, req, "dropping resent queued req");
+                GOTO(err_exp, rc = 0);
+        }
 
         /* XXX O(n^2) */
         list_for_each(tmp, &obd->obd_recovery_queue) {
@@ -1384,6 +1428,15 @@ int target_queue_recovery_request(struct ptlrpc_request *req,
                         inserted = 1;
                         break;
                 }
+
+                if (unlikely(lustre_msg_get_transno(reqiter->rq_reqmsg) ==
+                             transno)) {
+                        spin_unlock_bh(&obd->obd_processing_task_lock);
+                        DEBUG_REQ(D_ERROR, req, "dropping replay: transno "
+                                  "has been claimed by another client");
+                        target_exp_dequeue_req_replay(req);
+                        GOTO(err_exp, rc = 0);
+                }
         }
 
         if (!inserted) {
@@ -1409,6 +1462,13 @@ int target_queue_recovery_request(struct ptlrpc_request *req,
 
         process_recovery_queue(obd);
         return 0;
+
+err_exp:
+        class_export_put(req->rq_export);
+err_free:
+        OBD_FREE(reqmsg, req->rq_reqlen);
+        OBD_FREE(saved_req, sizeof(*saved_req));
+        return rc;
 }
 
 struct obd_device * target_req2obd(struct ptlrpc_request *req)

diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c
index 63d52c2..f6fbf45 100644 (file)
--- a/lustre/obdclass/genops.c
+++ b/lustre/obdclass/genops.c
@@ -640,6 +640,7 @@ void class_export_destroy(struct obd_export *exp)
                 ptlrpc_put_connection_superhack(exp->exp_connection);
 
         LASSERT(list_empty(&exp->exp_outstanding_replies));
+        LASSERT(list_empty(&exp->exp_req_replay_queue));
         obd_destroy_export(exp);
 
         OBD_FREE_RCU(exp, sizeof(*exp), &exp->exp_handle);
@@ -664,6 +665,7 @@ struct obd_export *class_new_export(struct obd_device *obd,
         atomic_set(&export->exp_rpc_count, 0);
         export->exp_obd = obd;
         CFS_INIT_LIST_HEAD(&export->exp_outstanding_replies);
+        CFS_INIT_LIST_HEAD(&export->exp_req_replay_queue);
         /* XXX this should be in LDLM init */
         CFS_INIT_LIST_HEAD(&export->exp_ldlm_data.led_held_locks);
         spin_lock_init(&export->exp_ldlm_data.led_lock);

diff --git a/lustre/tests/replay-single.sh b/lustre/tests/replay-single.sh
index 5142e45..b60f032 100755 (executable)
--- a/lustre/tests/replay-single.sh
+++ b/lustre/tests/replay-single.sh
@@ -1367,6 +1367,19 @@ test_61c() {
 }
 run_test 61c "test race mds llog sync vs llog cleanup"
 
+test_62() { # Bug 15756 - don't mis-drop resent replay
+    replay_barrier mds
+    createmany -o $DIR/$tdir/$tfile- 25
+#define OBD_FAIL_TGT_REPLAY_DROP         0x707
+    do_facet mds "sysctl -w lustre.fail_loc=0x80000707"
+    facet_failover mds
+    df $MOUNT || return 1
+    do_facet mds "sysctl -w lustre.fail_loc=0"
+    unlinkmany $DIR/$tdir/$tfile- 25 || return 2
+    return 0
+}
+run_test 62 "don't mis-drop resent replay"
+
 #Adaptive Timeouts (bug 3055)
 AT_MAX_SET=0