Whamcloud - gitweb
LU-11128 ptlrpc: new request vs disconnect race
[fs/lustre-release.git] / lustre / ptlrpc / import.c
index 2e7ac0e..8ca1dec 100644 (file)
@@ -256,7 +256,7 @@ static time64_t ptlrpc_inflight_deadline(struct ptlrpc_request *req,
         return dl - now;
 }
 
-static unsigned int ptlrpc_inflight_timeout(struct obd_import *imp)
+static time64_t ptlrpc_inflight_timeout(struct obd_import *imp)
 {
        time64_t now = ktime_get_real_seconds();
        struct list_head *tmp, *n;
@@ -305,16 +305,16 @@ void ptlrpc_invalidate_import(struct obd_import *imp)
                  * out. Use obd_timeout if calculated value is smaller
                 * than it.
                 */
-                if (!OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_LONG_REPL_UNLINK)) {
-                        timeout = ptlrpc_inflight_timeout(imp);
-                        timeout += timeout / 3;
+               if (!OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_LONG_REPL_UNLINK)) {
+                       timeout = ptlrpc_inflight_timeout(imp);
+                       timeout += div_u64(timeout, 3);
 
-                        if (timeout == 0)
-                                timeout = obd_timeout;
-                } else {
-                        /* decrease the interval to increase race condition */
-                        timeout = 1;
-                }
+                       if (timeout == 0)
+                               timeout = obd_timeout;
+               } else {
+                       /* decrease the interval to increase race condition */
+                       timeout = 1;
+               }
 
                CDEBUG(D_RPCTRACE, "Sleeping %llds for inflight to error out\n",
                       timeout);
@@ -791,9 +791,9 @@ static int ptlrpc_busy_reconnect(int rc)
 }
 
 static int ptlrpc_connect_set_flags(struct obd_import *imp,
-                                    struct obd_connect_data *ocd,
-                                    __u64 old_connect_flags,
-                                    struct obd_export *exp, int init_connect)
+                                   struct obd_connect_data *ocd,
+                                   __u64 old_connect_flags,
+                                   struct obd_export *exp, int init_connect)
 {
        static bool warned;
        struct client_obd *cli = &imp->imp_obd->u.cli;
@@ -836,13 +836,13 @@ static int ptlrpc_connect_set_flags(struct obd_import *imp,
                 * for algorithms we understand. The server masked off
                 * the checksum types it doesn't support */
                if ((ocd->ocd_cksum_types &
-                    cksum_types_supported_client()) == 0) {
+                    obd_cksum_types_supported_client()) == 0) {
                        LCONSOLE_ERROR("The negotiation of the checksum "
                                       "alogrithm to use with server %s "
                                       "failed (%x/%x)\n",
                                       obd2cli_tgt(imp->imp_obd),
                                       ocd->ocd_cksum_types,
-                                      cksum_types_supported_client());
+                                      obd_cksum_types_supported_client());
                        return -EPROTO;
                } else {
                        cli->cl_supp_cksum_types = ocd->ocd_cksum_types;
@@ -852,7 +852,8 @@ static int ptlrpc_connect_set_flags(struct obd_import *imp,
                 * Enforce ADLER for backward compatibility*/
                cli->cl_supp_cksum_types = OBD_CKSUM_ADLER;
        }
-       cli->cl_cksum_type = cksum_type_select(cli->cl_supp_cksum_types);
+       cli->cl_cksum_type = obd_cksum_type_select(imp->imp_obd->obd_name,
+                                                  cli->cl_supp_cksum_types);
 
        if (ocd->ocd_connect_flags & OBD_CONNECT_BRW_SIZE)
                cli->cl_max_pages_per_rpc =
@@ -967,6 +968,21 @@ static int ptlrpc_connect_interpret(const struct lu_env *env,
        }
 
        if (rc) {
+               struct ptlrpc_request *free_req;
+               struct ptlrpc_request *tmp;
+
+               /* abort all delayed requests initiated connection */
+               list_for_each_entry_safe(free_req, tmp, &imp->imp_delayed_list,
+                                        rq_list) {
+                       spin_lock(&free_req->rq_lock);
+                       if (free_req->rq_no_resend) {
+                               free_req->rq_err = 1;
+                               free_req->rq_status = -EIO;
+                               ptlrpc_client_wake_req(free_req);
+                       }
+                       spin_unlock(&free_req->rq_lock);
+               }
+
                /* if this reconnect to busy export - not need select new target
                 * for connecting*/
                imp->imp_force_reconnect = ptlrpc_busy_reconnect(rc);
@@ -1445,6 +1461,7 @@ int ptlrpc_import_recovery_state_machine(struct obd_import *imp)
                                           "using this service will fail.\n",
                                           imp->imp_obd->obd_name, target_len,
                                           target_start);
+                       LASSERTF(!obd_lbug_on_eviction, "LBUG upon eviction");
                 }
                 CDEBUG(D_HA, "evicted from %s@%s; invalidating\n",
                        obd2cli_tgt(imp->imp_obd),
@@ -1527,15 +1544,12 @@ out:
        RETURN(rc);
 }
 
-int ptlrpc_disconnect_import(struct obd_import *imp, int noclose)
+static struct ptlrpc_request *ptlrpc_disconnect_prep_req(struct obd_import *imp)
 {
        struct ptlrpc_request *req;
        int rq_opc, rc = 0;
        ENTRY;
 
-       if (imp->imp_obd->obd_force)
-               GOTO(set_state, rc);
-
        switch (imp->imp_connect_op) {
        case OST_CONNECT:
                rq_opc = OST_DISCONNECT;
@@ -1552,9 +1566,46 @@ int ptlrpc_disconnect_import(struct obd_import *imp, int noclose)
                       "(connect_op %d): rc = %d\n",
                       imp->imp_obd->obd_name, obd2cli_tgt(imp->imp_obd),
                       imp->imp_connect_op, rc);
-               RETURN(rc);
+               RETURN(ERR_PTR(rc));
        }
 
+       req = ptlrpc_request_alloc_pack(imp, &RQF_MDS_DISCONNECT,
+                                       LUSTRE_OBD_VERSION, rq_opc);
+       if (req == NULL)
+               RETURN(NULL);
+
+       /* We are disconnecting, do not retry a failed DISCONNECT rpc if
+        * it fails.  We can get through the above with a down server
+        * if the client doesn't know the server is gone yet. */
+       req->rq_no_resend = 1;
+
+       /* We want client umounts to happen quickly, no matter the
+          server state... */
+       req->rq_timeout = min_t(int, req->rq_timeout,
+                               INITIAL_CONNECT_TIMEOUT);
+
+       IMPORT_SET_STATE(imp, LUSTRE_IMP_CONNECTING);
+       req->rq_send_state =  LUSTRE_IMP_CONNECTING;
+       ptlrpc_request_set_replen(req);
+
+       RETURN(req);
+}
+
+int ptlrpc_disconnect_import(struct obd_import *imp, int noclose)
+{
+       struct ptlrpc_request *req;
+       int rc = 0;
+       ENTRY;
+
+       if (imp->imp_obd->obd_force)
+               GOTO(set_state, rc);
+
+       /* probably the import has been disconnected already being idle */
+       spin_lock(&imp->imp_lock);
+       if (imp->imp_state == LUSTRE_IMP_IDLE)
+               GOTO(out, rc);
+       spin_unlock(&imp->imp_lock);
+
        if (ptlrpc_import_in_recovery(imp)) {
                struct l_wait_info lwi;
                long timeout_jiffies;
@@ -1587,25 +1638,11 @@ int ptlrpc_disconnect_import(struct obd_import *imp, int noclose)
                GOTO(out, rc);
        spin_unlock(&imp->imp_lock);
 
-        req = ptlrpc_request_alloc_pack(imp, &RQF_MDS_DISCONNECT,
-                                        LUSTRE_OBD_VERSION, rq_opc);
-        if (req) {
-                /* We are disconnecting, do not retry a failed DISCONNECT rpc if
-                 * it fails.  We can get through the above with a down server
-                 * if the client doesn't know the server is gone yet. */
-                req->rq_no_resend = 1;
-
-                /* We want client umounts to happen quickly, no matter the
-                   server state... */
-                req->rq_timeout = min_t(int, req->rq_timeout,
-                                        INITIAL_CONNECT_TIMEOUT);
-
-                IMPORT_SET_STATE(imp, LUSTRE_IMP_CONNECTING);
-                req->rq_send_state =  LUSTRE_IMP_CONNECTING;
-                ptlrpc_request_set_replen(req);
-                rc = ptlrpc_queue_wait(req);
-                ptlrpc_req_finished(req);
-        }
+       req = ptlrpc_disconnect_prep_req(imp);
+       if (IS_ERR(req))
+               GOTO(set_state, rc = PTR_ERR(req));
+       rc = ptlrpc_queue_wait(req);
+       ptlrpc_req_finished(req);
 
 set_state:
        spin_lock(&imp->imp_lock);
@@ -1623,6 +1660,78 @@ out:
 }
 EXPORT_SYMBOL(ptlrpc_disconnect_import);
 
+static int ptlrpc_disconnect_idle_interpret(const struct lu_env *env,
+                                           struct ptlrpc_request *req,
+                                           void *data, int rc)
+{
+       struct obd_import *imp = req->rq_import;
+       int connect = 0;
+
+       DEBUG_REQ(D_HA, req, "inflight=%d, refcount=%d: rc = %d\n",
+                 atomic_read(&imp->imp_inflight),
+                 atomic_read(&imp->imp_refcount), rc);
+
+       spin_lock(&imp->imp_lock);
+       /* DISCONNECT reply can be late and another connection can just
+        * be initiated. so we have to abort disconnection. */
+       if (req->rq_import_generation == imp->imp_generation &&
+           imp->imp_state != LUSTRE_IMP_CLOSED) {
+               LASSERTF(imp->imp_state == LUSTRE_IMP_CONNECTING,
+                        "%s\n", ptlrpc_import_state_name(imp->imp_state));
+               imp->imp_state = LUSTRE_IMP_IDLE;
+               memset(&imp->imp_remote_handle, 0,
+                      sizeof(imp->imp_remote_handle));
+               /* take our DISCONNECT into account */
+               if (atomic_read(&imp->imp_inflight) > 1) {
+                       imp->imp_generation++;
+                       imp->imp_initiated_at = imp->imp_generation;
+                       IMPORT_SET_STATE_NOLOCK(imp, LUSTRE_IMP_NEW);
+                       connect = 1;
+               }
+       }
+       spin_unlock(&imp->imp_lock);
+
+       if (connect) {
+               rc = ptlrpc_connect_import(imp);
+               if (rc >= 0)
+                       ptlrpc_pinger_add_import(imp);
+       }
+
+       return 0;
+}
+
+int ptlrpc_disconnect_and_idle_import(struct obd_import *imp)
+{
+       struct ptlrpc_request *req;
+       ENTRY;
+
+       if (imp->imp_obd->obd_force)
+               RETURN(0);
+
+       if (ptlrpc_import_in_recovery(imp))
+               RETURN(0);
+
+       spin_lock(&imp->imp_lock);
+       if (imp->imp_state != LUSTRE_IMP_FULL) {
+               spin_unlock(&imp->imp_lock);
+               RETURN(0);
+       }
+       spin_unlock(&imp->imp_lock);
+
+       req = ptlrpc_disconnect_prep_req(imp);
+       if (IS_ERR(req))
+               RETURN(PTR_ERR(req));
+
+       CDEBUG_LIMIT(imp->imp_idle_debug, "%s: disconnect after %llus idle\n",
+                    imp->imp_obd->obd_name,
+                    ktime_get_real_seconds() - imp->imp_last_reply_time);
+       req->rq_interpret_reply = ptlrpc_disconnect_idle_interpret;
+       ptlrpcd_add_req(req);
+
+       RETURN(0);
+}
+EXPORT_SYMBOL(ptlrpc_disconnect_and_idle_import);
+
 void ptlrpc_cleanup_imp(struct obd_import *imp)
 {
        ENTRY;