Whamcloud - gitweb
LU-7434 ptlrpc: lost bulk leads to a hang
[fs/lustre-release.git] / lustre / target / tgt_handler.c
index 21cecc2..ccf77a8 100644 (file)
@@ -604,6 +604,52 @@ static struct tgt_handler *tgt_handler_find_check(struct ptlrpc_request *req)
        RETURN(h);
 }
 
+static int process_req_last_xid(struct ptlrpc_request *req)
+{
+       __u64   last_xid;
+       ENTRY;
+
+       /* check request's xid is consistent with export's last_xid */
+       last_xid = lustre_msg_get_last_xid(req->rq_reqmsg);
+       if (last_xid > req->rq_export->exp_last_xid)
+               req->rq_export->exp_last_xid = last_xid;
+
+       if (req->rq_xid == 0 ||
+           (req->rq_xid <= req->rq_export->exp_last_xid)) {
+               DEBUG_REQ(D_ERROR, req, "Unexpected xid %llx vs. "
+                         "last_xid %llx\n", req->rq_xid,
+                         req->rq_export->exp_last_xid);
+               /* Some request is allowed to be sent during replay,
+                * such as OUT update requests, FLD requests, so it
+                * is possible that replay requests has smaller XID
+                * than the exp_last_xid.
+                *
+                * Some non-replay requests may have smaller XID as
+                * well:
+                *
+                * - Client send a no_resend RPC, like statfs;
+                * - The RPC timedout (or some other error) on client,
+                *   then it's removed from the unreplied list;
+                * - Client send some other request to bump the
+                *   exp_last_xid on server;
+                * - The former RPC got chance to be processed;
+                */
+               if (!(lustre_msg_get_flags(req->rq_reqmsg) & MSG_REPLAY))
+                       RETURN(-EPROTO);
+       }
+
+       /* try to release in-memory reply data */
+       if (tgt_is_multimodrpcs_client(req->rq_export)) {
+               tgt_handle_received_xid(req->rq_export,
+                               lustre_msg_get_last_xid(req->rq_reqmsg));
+               if (!(lustre_msg_get_flags(req->rq_reqmsg) &
+                     (MSG_RESENT | MSG_REPLAY)))
+                       tgt_handle_tag(req->rq_export,
+                                      lustre_msg_get_tag(req->rq_reqmsg));
+       }
+       RETURN(0);
+}
+
 int tgt_request_handle(struct ptlrpc_request *req)
 {
        struct tgt_session_info *tsi = tgt_ses_info(req->rq_svc_thread->t_env);
@@ -613,8 +659,9 @@ int tgt_request_handle(struct ptlrpc_request *req)
        struct lu_target        *tgt;
        int                      request_fail_id = 0;
        __u32                    opc = lustre_msg_get_opc(msg);
+       struct obd_device       *obd;
        int                      rc;
-
+       bool                     is_connect = false;
        ENTRY;
 
        /* Refill the context, to make sure all thread keys are allocated */
@@ -628,6 +675,7 @@ int tgt_request_handle(struct ptlrpc_request *req)
         * target, otherwise that should be connect operation */
        if (opc == MDS_CONNECT || opc == OST_CONNECT ||
            opc == MGS_CONNECT) {
+               is_connect = true;
                req_capsule_set(&req->rq_pill, &RQF_CONNECT);
                rc = target_handle_connect(req);
                if (rc != 0) {
@@ -671,20 +719,19 @@ int tgt_request_handle(struct ptlrpc_request *req)
                GOTO(out, rc);
        }
 
-       /* check request's xid is consistent with export's last_xid */
-       if (req->rq_export != NULL) {
-               __u64 last_xid = lustre_msg_get_last_xid(req->rq_reqmsg);
-               if (last_xid != 0)
-                       req->rq_export->exp_last_xid = last_xid;
-               if (req->rq_xid == 0 ||
-                   req->rq_xid <= req->rq_export->exp_last_xid) {
-                       DEBUG_REQ(D_ERROR, req,
-                                 "Unexpected xid %llx vs. last_xid %llx\n",
-                                 req->rq_xid, req->rq_export->exp_last_xid);
-#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 7, 93, 0)
-                       LBUG();
-#endif
-                       req->rq_status = -EPROTO;
+       /* Skip last_xid processing for the recovery thread, otherwise, the
+        * last_xid on same request could be processed twice: first time when
+        * processing the incoming request, second time when the request is
+        * being processed by recovery thread. */
+       obd = class_exp2obd(req->rq_export);
+       if (is_connect) {
+               /* reset the exp_last_xid on each connection. */
+               req->rq_export->exp_last_xid = 0;
+       } else if (obd->obd_recovery_data.trd_processing_task !=
+                  current_pid()) {
+               rc = process_req_last_xid(req);
+               if (rc) {
+                       req->rq_status = rc;
                        rc = ptlrpc_error(req);
                        GOTO(out, rc);
                }
@@ -693,16 +740,6 @@ int tgt_request_handle(struct ptlrpc_request *req)
        request_fail_id = tgt->lut_request_fail_id;
        tsi->tsi_reply_fail_id = tgt->lut_reply_fail_id;
 
-       /* try to release in-memory reply data */
-       if (tgt_is_multimodrpcs_client(req->rq_export)) {
-               tgt_handle_received_xid(req->rq_export,
-                               lustre_msg_get_last_xid(req->rq_reqmsg));
-               if (!(lustre_msg_get_flags(req->rq_reqmsg) &
-                     (MSG_RESENT | MSG_REPLAY)))
-                       tgt_handle_tag(req->rq_export,
-                                      lustre_msg_get_tag(req->rq_reqmsg));
-       }
-
        h = tgt_handler_find_check(req);
        if (IS_ERR(h)) {
                req->rq_status = PTR_ERR(h);
@@ -1248,6 +1285,7 @@ int tgt_sync(const struct lu_env *env, struct lu_target *tgt,
                   tgt->lut_obd->obd_last_committed) {
                rc = dt_object_sync(env, obj, start, end);
        }
+       atomic_inc(&tgt->lut_sync_count);
 
        RETURN(rc);
 }
@@ -1256,14 +1294,27 @@ EXPORT_SYMBOL(tgt_sync);
  * Unified target DLM handlers.
  */
 
-/* Ensure that data and metadata are synced to the disk when lock is cancelled
- * (if requested) */
+/**
+ * Unified target BAST
+ *
+ * Ensure data and metadata are synced to disk when lock is canceled if Sync on
+ * Cancel (SOC) is enabled. If it's extent lock, normally sync obj is enough,
+ * but if it's cross-MDT lock, because remote object version is not set, a
+ * filesystem sync is needed.
+ *
+ * \param lock server side lock
+ * \param desc lock desc
+ * \param data ldlm_cb_set_arg
+ * \param flag indicates whether this cancelling or blocking callback
+ * \retval     0 on success
+ * \retval     negative number on error
+ */
 static int tgt_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc,
                            void *data, int flag)
 {
        struct lu_env            env;
        struct lu_target        *tgt;
-       struct dt_object        *obj;
+       struct dt_object        *obj = NULL;
        struct lu_fid            fid;
        int                      rc = 0;
 
@@ -1278,10 +1329,12 @@ static int tgt_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc,
        }
 
        if (flag == LDLM_CB_CANCELING &&
-           (lock->l_granted_mode & (LCK_PW | LCK_GROUP)) &&
+           (lock->l_granted_mode & (LCK_EX | LCK_PW | LCK_GROUP)) &&
            (tgt->lut_sync_lock_cancel == ALWAYS_SYNC_ON_CANCEL ||
             (tgt->lut_sync_lock_cancel == BLOCKING_SYNC_ON_CANCEL &&
-             lock->l_flags & LDLM_FL_CBPENDING))) {
+             ldlm_is_cbpending(lock))) &&
+           ((exp_connect_flags(lock->l_export) & OBD_CONNECT_MDS_MDS) ||
+            lock->l_resource->lr_type == LDLM_EXTENT)) {
                __u64 start = 0;
                __u64 end = OBD_OBJECT_EOF;
 
@@ -1291,14 +1344,15 @@ static int tgt_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc,
 
                ost_fid_from_resid(&fid, &lock->l_resource->lr_name,
                                   tgt->lut_lsd.lsd_osd_index);
-               obj = dt_locate(&env, tgt->lut_bottom, &fid);
-               if (IS_ERR(obj))
-                       GOTO(err_env, rc = PTR_ERR(obj));
-
-               if (!dt_object_exists(obj))
-                       GOTO(err_put, rc = -ENOENT);
 
                if (lock->l_resource->lr_type == LDLM_EXTENT) {
+                       obj = dt_locate(&env, tgt->lut_bottom, &fid);
+                       if (IS_ERR(obj))
+                               GOTO(err_env, rc = PTR_ERR(obj));
+
+                       if (!dt_object_exists(obj))
+                               GOTO(err_put, rc = -ENOENT);
+
                        start = lock->l_policy_data.l_extent.start;
                        end = lock->l_policy_data.l_extent.end;
                }
@@ -1312,7 +1366,8 @@ static int tgt_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc,
                               lock->l_policy_data.l_extent.end, rc);
                }
 err_put:
-               lu_object_put(&env, &obj->do_lu);
+               if (obj != NULL)
+                       lu_object_put(&env, &obj->do_lu);
 err_env:
                lu_env_fini(&env);
        }
@@ -1512,11 +1567,15 @@ EXPORT_SYMBOL(tgt_register_lfsck_in_notify);
 
 static int (*tgt_lfsck_query)(const struct lu_env *env,
                              struct dt_device *key,
-                             struct lfsck_request *lr) = NULL;
+                             struct lfsck_request *req,
+                             struct lfsck_reply *rep,
+                             struct lfsck_query *que) = NULL;
 
 void tgt_register_lfsck_query(int (*query)(const struct lu_env *,
                                           struct dt_device *,
-                                          struct lfsck_request *))
+                                          struct lfsck_request *,
+                                          struct lfsck_reply *,
+                                          struct lfsck_query *))
 {
        tgt_lfsck_query = query;
 }
@@ -1555,8 +1614,8 @@ static int tgt_handle_lfsck_query(struct tgt_session_info *tsi)
        if (reply == NULL)
                RETURN(-ENOMEM);
 
-       rc = tgt_lfsck_query(tsi->tsi_env, tsi->tsi_tgt->lut_bottom, request);
-       reply->lr_status = rc;
+       rc = tgt_lfsck_query(tsi->tsi_env, tsi->tsi_tgt->lut_bottom,
+                            request, reply, NULL);
 
        RETURN(rc < 0 ? rc : 0);
 }
@@ -1918,7 +1977,8 @@ int tgt_brw_read(struct tgt_session_info *tsi)
        /* Check if client was evicted while we were doing i/o before touching
         * network */
        if (likely(rc == 0 &&
-                  !CFS_FAIL_PRECHECK(OBD_FAIL_PTLRPC_CLIENT_BULK_CB2))) {
+                  !CFS_FAIL_PRECHECK(OBD_FAIL_PTLRPC_CLIENT_BULK_CB2) &&
+                  !CFS_FAIL_CHECK(OBD_FAIL_PTLRPC_DROP_BULK))) {
                rc = target_bulk_io(exp, desc, &lwi);
                no_reply = rc != 0;
        }