Whamcloud - gitweb
LU-7434 ptlrpc: lost bulk leads to a hang
[fs/lustre-release.git] / lustre / target / tgt_handler.c
index 2e369a6..ccf77a8 100644 (file)
@@ -21,7 +21,7 @@
  * GPL HEADER END
  */
 /*
- * Copyright (c) 2011, 2014, Intel Corporation.
+ * Copyright (c) 2012, 2015, Intel Corporation.
  */
 /*
  * lustre/target/tgt_handler.c
@@ -389,6 +389,7 @@ static int tgt_handle_request0(struct tgt_session_info *tsi,
        case SEC_CTX_INIT:
        case SEC_CTX_INIT_CONT:
        case SEC_CTX_FINI:
+               CFS_FAIL_TIMEOUT(OBD_FAIL_SEC_CTX_HDL_PAUSE, cfs_fail_val);
                GOTO(out, rc = 0);
        }
 
@@ -405,6 +406,9 @@ static int tgt_handle_request0(struct tgt_session_info *tsi,
         */
        if (OBD_FAIL_CHECK_ORSET(h->th_fail_id, OBD_FAIL_ONCE))
                RETURN(0);
+       if (unlikely(lustre_msg_get_opc(req->rq_reqmsg) == MDS_REINT &&
+                    OBD_FAIL_CHECK(OBD_FAIL_MDS_REINT_MULTI_NET)))
+               RETURN(0);
 
        rc = tgt_request_preprocess(tsi, h, req);
        /* pack reply if reply format is fixed */
@@ -600,6 +604,52 @@ static struct tgt_handler *tgt_handler_find_check(struct ptlrpc_request *req)
        RETURN(h);
 }
 
+static int process_req_last_xid(struct ptlrpc_request *req)
+{
+       __u64   last_xid;
+       ENTRY;
+
+       /* check request's xid is consistent with export's last_xid */
+       last_xid = lustre_msg_get_last_xid(req->rq_reqmsg);
+       if (last_xid > req->rq_export->exp_last_xid)
+               req->rq_export->exp_last_xid = last_xid;
+
+       if (req->rq_xid == 0 ||
+           (req->rq_xid <= req->rq_export->exp_last_xid)) {
+               DEBUG_REQ(D_ERROR, req, "Unexpected xid %llx vs. "
+                         "last_xid %llx\n", req->rq_xid,
+                         req->rq_export->exp_last_xid);
+               /* Some request is allowed to be sent during replay,
+                * such as OUT update requests, FLD requests, so it
+                * is possible that replay requests has smaller XID
+                * than the exp_last_xid.
+                *
+                * Some non-replay requests may have smaller XID as
+                * well:
+                *
+                * - Client send a no_resend RPC, like statfs;
+                * - The RPC timedout (or some other error) on client,
+                *   then it's removed from the unreplied list;
+                * - Client send some other request to bump the
+                *   exp_last_xid on server;
+                * - The former RPC got chance to be processed;
+                */
+               if (!(lustre_msg_get_flags(req->rq_reqmsg) & MSG_REPLAY))
+                       RETURN(-EPROTO);
+       }
+
+       /* try to release in-memory reply data */
+       if (tgt_is_multimodrpcs_client(req->rq_export)) {
+               tgt_handle_received_xid(req->rq_export,
+                               lustre_msg_get_last_xid(req->rq_reqmsg));
+               if (!(lustre_msg_get_flags(req->rq_reqmsg) &
+                     (MSG_RESENT | MSG_REPLAY)))
+                       tgt_handle_tag(req->rq_export,
+                                      lustre_msg_get_tag(req->rq_reqmsg));
+       }
+       RETURN(0);
+}
+
 int tgt_request_handle(struct ptlrpc_request *req)
 {
        struct tgt_session_info *tsi = tgt_ses_info(req->rq_svc_thread->t_env);
@@ -609,8 +659,9 @@ int tgt_request_handle(struct ptlrpc_request *req)
        struct lu_target        *tgt;
        int                      request_fail_id = 0;
        __u32                    opc = lustre_msg_get_opc(msg);
+       struct obd_device       *obd;
        int                      rc;
-
+       bool                     is_connect = false;
        ENTRY;
 
        /* Refill the context, to make sure all thread keys are allocated */
@@ -624,6 +675,7 @@ int tgt_request_handle(struct ptlrpc_request *req)
         * target, otherwise that should be connect operation */
        if (opc == MDS_CONNECT || opc == OST_CONNECT ||
            opc == MGS_CONNECT) {
+               is_connect = true;
                req_capsule_set(&req->rq_pill, &RQF_CONNECT);
                rc = target_handle_connect(req);
                if (rc != 0) {
@@ -667,20 +719,19 @@ int tgt_request_handle(struct ptlrpc_request *req)
                GOTO(out, rc);
        }
 
-       /* check request's xid is consistent with export's last_xid */
-       if (req->rq_export != NULL) {
-               __u64 last_xid = lustre_msg_get_last_xid(req->rq_reqmsg);
-               if (last_xid != 0)
-                       req->rq_export->exp_last_xid = last_xid;
-               if (req->rq_xid == 0 ||
-                   req->rq_xid <= req->rq_export->exp_last_xid) {
-                       DEBUG_REQ(D_ERROR, req,
-                                 "Unexpected xid %llx vs. last_xid %llx\n",
-                                 req->rq_xid, req->rq_export->exp_last_xid);
-#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 7, 93, 0)
-                       LBUG();
-#endif
-                       req->rq_status = -EPROTO;
+       /* Skip last_xid processing for the recovery thread, otherwise, the
+        * last_xid on same request could be processed twice: first time when
+        * processing the incoming request, second time when the request is
+        * being processed by recovery thread. */
+       obd = class_exp2obd(req->rq_export);
+       if (is_connect) {
+               /* reset the exp_last_xid on each connection. */
+               req->rq_export->exp_last_xid = 0;
+       } else if (obd->obd_recovery_data.trd_processing_task !=
+                  current_pid()) {
+               rc = process_req_last_xid(req);
+               if (rc) {
+                       req->rq_status = rc;
                        rc = ptlrpc_error(req);
                        GOTO(out, rc);
                }
@@ -689,16 +740,6 @@ int tgt_request_handle(struct ptlrpc_request *req)
        request_fail_id = tgt->lut_request_fail_id;
        tsi->tsi_reply_fail_id = tgt->lut_reply_fail_id;
 
-       /* try to release in-memory reply data */
-       if (tgt_is_multimodrpcs_client(req->rq_export)) {
-               tgt_handle_received_xid(req->rq_export,
-                               lustre_msg_get_last_xid(req->rq_reqmsg));
-               if (!(lustre_msg_get_flags(req->rq_reqmsg) &
-                     (MSG_RESENT | MSG_REPLAY)))
-                       tgt_handle_tag(req->rq_export,
-                                      lustre_msg_get_tag(req->rq_reqmsg));
-       }
-
        h = tgt_handler_find_check(req);
        if (IS_ERR(h)) {
                req->rq_status = PTR_ERR(h);
@@ -931,6 +972,16 @@ int tgt_connect_check_sptlrpc(struct ptlrpc_request *req, struct obd_export *exp
                spin_lock(&exp->exp_lock);
                exp->exp_sp_peer = req->rq_sp_from;
                exp->exp_flvr = flvr;
+
+               /* when on mgs, if no restriction is set, or if client
+                * is loopback, allow any flavor */
+               if ((strcmp(exp->exp_obd->obd_type->typ_name,
+                          LUSTRE_MGS_NAME) == 0) &&
+                    (exp->exp_flvr.sf_rpc == SPTLRPC_FLVR_NULL ||
+                     LNET_NETTYP(LNET_NIDNET(exp->exp_connection->c_peer.nid))
+                     == LOLND))
+                       exp->exp_flvr.sf_rpc = SPTLRPC_FLVR_ANY;
+
                if (exp->exp_flvr.sf_rpc != SPTLRPC_FLVR_ANY &&
                    exp->exp_flvr.sf_rpc != req->rq_flvr.sf_rpc) {
                        CERROR("%s: unauthorized rpc flavor %x from %s, "
@@ -1058,6 +1109,35 @@ int tgt_obd_log_cancel(struct tgt_session_info *tsi)
        return err_serious(-EOPNOTSUPP);
 }
 
+int tgt_send_buffer(struct tgt_session_info *tsi, struct lu_rdbuf *rdbuf)
+{
+       struct tgt_thread_info  *tti = tgt_th_info(tsi->tsi_env);
+       struct ptlrpc_request   *req = tgt_ses_req(tsi);
+       struct obd_export       *exp = req->rq_export;
+       struct ptlrpc_bulk_desc *desc;
+       struct l_wait_info      *lwi = &tti->tti_u.update.tti_wait_info;
+       int                      i;
+       int                      rc;
+
+       ENTRY;
+
+       desc = ptlrpc_prep_bulk_exp(req, rdbuf->rb_nbufs, 1,
+                                 PTLRPC_BULK_PUT_SOURCE | PTLRPC_BULK_BUF_KVEC,
+                                   MDS_BULK_PORTAL, &ptlrpc_bulk_kvec_ops);
+       if (desc == NULL)
+               RETURN(-ENOMEM);
+
+       for (i = 0; i < rdbuf->rb_nbufs; i++)
+               desc->bd_frag_ops->add_iov_frag(desc,
+                                       rdbuf->rb_bufs[i].lb_buf,
+                                       rdbuf->rb_bufs[i].lb_len);
+
+       rc = target_bulk_io(exp, desc, lwi);
+       ptlrpc_free_bulk(desc);
+       RETURN(rc);
+}
+EXPORT_SYMBOL(tgt_send_buffer);
+
 int tgt_sendpage(struct tgt_session_info *tsi, struct lu_rdpg *rdpg, int nob)
 {
        struct tgt_thread_info  *tti = tgt_th_info(tsi->tsi_env);
@@ -1205,6 +1285,7 @@ int tgt_sync(const struct lu_env *env, struct lu_target *tgt,
                   tgt->lut_obd->obd_last_committed) {
                rc = dt_object_sync(env, obj, start, end);
        }
+       atomic_inc(&tgt->lut_sync_count);
 
        RETURN(rc);
 }
@@ -1213,14 +1294,27 @@ EXPORT_SYMBOL(tgt_sync);
  * Unified target DLM handlers.
  */
 
-/* Ensure that data and metadata are synced to the disk when lock is cancelled
- * (if requested) */
+/**
+ * Unified target BAST
+ *
+ * Ensure data and metadata are synced to disk when lock is canceled if Sync on
+ * Cancel (SOC) is enabled. If it's extent lock, normally sync obj is enough,
+ * but if it's cross-MDT lock, because remote object version is not set, a
+ * filesystem sync is needed.
+ *
+ * \param lock server side lock
+ * \param desc lock desc
+ * \param data ldlm_cb_set_arg
+ * \param flag indicates whether this cancelling or blocking callback
+ * \retval     0 on success
+ * \retval     negative number on error
+ */
 static int tgt_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc,
                            void *data, int flag)
 {
        struct lu_env            env;
        struct lu_target        *tgt;
-       struct dt_object        *obj;
+       struct dt_object        *obj = NULL;
        struct lu_fid            fid;
        int                      rc = 0;
 
@@ -1235,10 +1329,12 @@ static int tgt_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc,
        }
 
        if (flag == LDLM_CB_CANCELING &&
-           (lock->l_granted_mode & (LCK_PW | LCK_GROUP)) &&
+           (lock->l_granted_mode & (LCK_EX | LCK_PW | LCK_GROUP)) &&
            (tgt->lut_sync_lock_cancel == ALWAYS_SYNC_ON_CANCEL ||
             (tgt->lut_sync_lock_cancel == BLOCKING_SYNC_ON_CANCEL &&
-             lock->l_flags & LDLM_FL_CBPENDING))) {
+             ldlm_is_cbpending(lock))) &&
+           ((exp_connect_flags(lock->l_export) & OBD_CONNECT_MDS_MDS) ||
+            lock->l_resource->lr_type == LDLM_EXTENT)) {
                __u64 start = 0;
                __u64 end = OBD_OBJECT_EOF;
 
@@ -1248,14 +1344,15 @@ static int tgt_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc,
 
                ost_fid_from_resid(&fid, &lock->l_resource->lr_name,
                                   tgt->lut_lsd.lsd_osd_index);
-               obj = dt_locate(&env, tgt->lut_bottom, &fid);
-               if (IS_ERR(obj))
-                       GOTO(err_env, rc = PTR_ERR(obj));
-
-               if (!dt_object_exists(obj))
-                       GOTO(err_put, rc = -ENOENT);
 
                if (lock->l_resource->lr_type == LDLM_EXTENT) {
+                       obj = dt_locate(&env, tgt->lut_bottom, &fid);
+                       if (IS_ERR(obj))
+                               GOTO(err_env, rc = PTR_ERR(obj));
+
+                       if (!dt_object_exists(obj))
+                               GOTO(err_put, rc = -ENOENT);
+
                        start = lock->l_policy_data.l_extent.start;
                        end = lock->l_policy_data.l_extent.end;
                }
@@ -1269,7 +1366,8 @@ static int tgt_blocking_ast(struct ldlm_lock *lock, struct ldlm_lock_desc *desc,
                               lock->l_policy_data.l_extent.end, rc);
                }
 err_put:
-               lu_object_put(&env, &obj->do_lu);
+               if (obj != NULL)
+                       lu_object_put(&env, &obj->do_lu);
 err_env:
                lu_env_fini(&env);
        }
@@ -1437,6 +1535,22 @@ TGT_LLOG_HDL_VAR(0,      LLOG_ORIGIN_HANDLE_CLOSE,       tgt_llog_close),
 };
 EXPORT_SYMBOL(tgt_llog_handlers);
 
+/*
+ * sec context handlers
+ */
+/* XXX: Implement based on mdt_sec_ctx_handle()? */
+static int tgt_sec_ctx_handle(struct tgt_session_info *tsi)
+{
+       return 0;
+}
+
+struct tgt_handler tgt_sec_ctx_handlers[] = {
+TGT_SEC_HDL_VAR(0,     SEC_CTX_INIT,           tgt_sec_ctx_handle),
+TGT_SEC_HDL_VAR(0,     SEC_CTX_INIT_CONT,      tgt_sec_ctx_handle),
+TGT_SEC_HDL_VAR(0,     SEC_CTX_FINI,           tgt_sec_ctx_handle),
+};
+EXPORT_SYMBOL(tgt_sec_ctx_handlers);
+
 int (*tgt_lfsck_in_notify)(const struct lu_env *env,
                           struct dt_device *key,
                           struct lfsck_request *lr,
@@ -1453,11 +1567,15 @@ EXPORT_SYMBOL(tgt_register_lfsck_in_notify);
 
 static int (*tgt_lfsck_query)(const struct lu_env *env,
                              struct dt_device *key,
-                             struct lfsck_request *lr) = NULL;
+                             struct lfsck_request *req,
+                             struct lfsck_reply *rep,
+                             struct lfsck_query *que) = NULL;
 
 void tgt_register_lfsck_query(int (*query)(const struct lu_env *,
                                           struct dt_device *,
-                                          struct lfsck_request *))
+                                          struct lfsck_request *,
+                                          struct lfsck_reply *,
+                                          struct lfsck_query *))
 {
        tgt_lfsck_query = query;
 }
@@ -1496,8 +1614,8 @@ static int tgt_handle_lfsck_query(struct tgt_session_info *tsi)
        if (reply == NULL)
                RETURN(-ENOMEM);
 
-       rc = tgt_lfsck_query(tsi->tsi_env, tsi->tsi_tgt->lut_bottom, request);
-       reply->lr_status = rc;
+       rc = tgt_lfsck_query(tsi->tsi_env, tsi->tsi_tgt->lut_bottom,
+                            request, reply, NULL);
 
        RETURN(rc < 0 ? rc : 0);
 }
@@ -1559,8 +1677,8 @@ int tgt_extent_lock(struct ldlm_namespace *ns, struct ldlm_res_id *res_id,
                    __u64 start, __u64 end, struct lustre_handle *lh,
                    int mode, __u64 *flags)
 {
-       ldlm_policy_data_t       policy;
-       int                      rc;
+       union ldlm_policy_data policy;
+       int rc;
 
        ENTRY;
 
@@ -1588,7 +1706,7 @@ int tgt_extent_lock(struct ldlm_namespace *ns, struct ldlm_res_id *res_id,
 }
 EXPORT_SYMBOL(tgt_extent_lock);
 
-void tgt_extent_unlock(struct lustre_handle *lh, ldlm_mode_t mode)
+void tgt_extent_unlock(struct lustre_handle *lh, enum ldlm_mode mode)
 {
        LASSERT(lustre_handle_is_used(lh));
        ldlm_lock_decref(lh, mode);
@@ -1597,7 +1715,7 @@ EXPORT_SYMBOL(tgt_extent_unlock);
 
 int tgt_brw_lock(struct ldlm_namespace *ns, struct ldlm_res_id *res_id,
                 struct obd_ioobj *obj, struct niobuf_remote *nb,
-                struct lustre_handle *lh, int mode)
+                struct lustre_handle *lh, enum ldlm_mode mode)
 {
        __u64                    flags = 0;
        int                      nrbufs = obj->ioo_bufcnt;
@@ -1622,7 +1740,7 @@ int tgt_brw_lock(struct ldlm_namespace *ns, struct ldlm_res_id *res_id,
 }
 
 void tgt_brw_unlock(struct obd_ioobj *obj, struct niobuf_remote *niob,
-                   struct lustre_handle *lh, int mode)
+                   struct lustre_handle *lh, enum ldlm_mode mode)
 {
        ENTRY;
 
@@ -1837,6 +1955,9 @@ int tgt_brw_read(struct tgt_session_info *tsi)
                        break;
                }
        }
+       if (OBD_FAIL_CHECK(OBD_FAIL_OST_READ_SIZE) &&
+           nob != cfs_fail_val)
+               rc = -E2BIG;
 
        if (body->oa.o_valid & OBD_MD_FLCKSUM) {
                cksum_type_t cksum_type =
@@ -1856,7 +1977,8 @@ int tgt_brw_read(struct tgt_session_info *tsi)
        /* Check if client was evicted while we were doing i/o before touching
         * network */
        if (likely(rc == 0 &&
-                  !CFS_FAIL_PRECHECK(OBD_FAIL_PTLRPC_CLIENT_BULK_CB2))) {
+                  !CFS_FAIL_PRECHECK(OBD_FAIL_PTLRPC_CLIENT_BULK_CB2) &&
+                  !CFS_FAIL_CHECK(OBD_FAIL_PTLRPC_DROP_BULK))) {
                rc = target_bulk_io(exp, desc, &lwi);
                no_reply = rc != 0;
        }