Whamcloud - gitweb
LU-1252 recovery: reduce reconnect time for IR
[fs/lustre-release.git] / lustre / ptlrpc / import.c
index b0a5292..7e68efb 100644 (file)
@@ -1,6 +1,4 @@
-/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
- * vim:expandtab:shiftwidth=8:tabstop=8:
- *
+/*
  * GPL HEADER START
  *
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -28,9 +26,8 @@
 /*
  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  * Use is subject to license terms.
- */
-/*
- * Copyright (c) 2011 Whamcloud, Inc.
+ *
+ * Copyright (c) 2011, Whamcloud, Inc.
  */
 /*
  * This file is part of Lustre, http://www.lustre.org/
@@ -121,8 +118,7 @@ int ptlrpc_init_import(struct obd_import *imp)
 EXPORT_SYMBOL(ptlrpc_init_import);
 
 #define UUID_STR "_UUID"
-static void deuuidify(char *uuid, const char *prefix, char **uuid_start,
-                      int *uuid_len)
+void deuuidify(char *uuid, const char *prefix, char **uuid_start, int *uuid_len)
 {
         *uuid_start = !prefix || strncmp(uuid, prefix, strlen(prefix))
                 ? uuid : uuid + strlen(prefix);
@@ -136,6 +132,7 @@ static void deuuidify(char *uuid, const char *prefix, char **uuid_start,
                     UUID_STR, strlen(UUID_STR)))
                 *uuid_len -= strlen(UUID_STR);
 }
+EXPORT_SYMBOL(deuuidify);
 
 /**
  * Returns true if import was FULL, false if import was already not
@@ -163,15 +160,15 @@ int ptlrpc_set_import_discon(struct obd_import *imp, __u32 conn_cnt)
                           &target_start, &target_len);
 
                 if (imp->imp_replayable) {
-                        LCONSOLE_WARN("%s: Connection to service %.*s via nid "
-                               "%s was lost; in progress operations using this "
-                               "service will wait for recovery to complete.\n",
+                        LCONSOLE_WARN("%s: Connection to %.*s (at %s) was "
+                               "lost; in progress operations using this "
+                               "service will wait for recovery to complete\n",
                                imp->imp_obd->obd_name, target_len, target_start,
                                libcfs_nid2str(imp->imp_connection->c_peer.nid));
                 } else {
-                        LCONSOLE_ERROR_MSG(0x166, "%s: Connection to service "
-                               "%.*s via nid %s was lost; in progress "
-                               "operations using this service will fail.\n",
+                        LCONSOLE_ERROR_MSG(0x166, "%s: Connection to "
+                               "%.*s (at %s) was lost; in progress "
+                               "operations using this service will fail\n",
                                imp->imp_obd->obd_name,
                                target_len, target_start,
                                libcfs_nid2str(imp->imp_connection->c_peer.nid));
@@ -461,7 +458,8 @@ static int import_select_connection(struct obd_import *imp)
 {
         struct obd_import_conn *imp_conn = NULL, *conn;
         struct obd_export *dlmexp;
-        int tried_all = 1;
+        char *target_start;
+        int target_len, tried_all = 1;
         ENTRY;
 
         cfs_spin_lock(&imp->imp_lock);
@@ -513,17 +511,16 @@ static int import_select_connection(struct obd_import *imp)
            state associated with the last connection attempt to drain before
            trying to reconnect on it.) */
         if (tried_all && (imp->imp_conn_list.next == &imp_conn->oic_item)) {
-                if (at_get(&imp->imp_at.iat_net_latency) <
-                    CONNECTION_SWITCH_MAX) {
-                        at_measured(&imp->imp_at.iat_net_latency,
-                                    at_get(&imp->imp_at.iat_net_latency) +
-                                    CONNECTION_SWITCH_INC);
-                }
-                LASSERT(imp_conn->oic_last_attempt);
-                CWARN("%s: tried all connections, increasing latency to %ds\n",
-                      imp->imp_obd->obd_name,
-                      at_get(&imp->imp_at.iat_net_latency));
-        }
+               struct adaptive_timeout *at = &imp->imp_at.iat_net_latency;
+               if (at_get(at) < CONNECTION_SWITCH_MAX) {
+                       at_measured(at, at_get(at) + CONNECTION_SWITCH_INC);
+                       if (at_get(at) > CONNECTION_SWITCH_MAX)
+                               at_reset(at, CONNECTION_SWITCH_MAX);
+               }
+               LASSERT(imp_conn->oic_last_attempt);
+               CDEBUG(D_HA, "%s: tried all connections, increasing latency "
+                       "to %ds\n", imp->imp_obd->obd_name, at_get(at));
+       }
 
         imp_conn->oic_last_attempt = cfs_time_current_64();
 
@@ -540,10 +537,17 @@ static int import_select_connection(struct obd_import *imp)
         class_export_put(dlmexp);
 
         if (imp->imp_conn_current != imp_conn) {
-                if (imp->imp_conn_current)
-                        CDEBUG(D_HA, "Changing connection for %s to %s/%s\n",
-                               imp->imp_obd->obd_name, imp_conn->oic_uuid.uuid,
+                if (imp->imp_conn_current) {
+                        deuuidify(obd2cli_tgt(imp->imp_obd), NULL,
+                                  &target_start, &target_len);
+
+                        CDEBUG(D_HA, "%s: Connection changing to"
+                               " %.*s (at %s)\n",
+                               imp->imp_obd->obd_name,
+                               target_len, target_start,
                                libcfs_nid2str(imp_conn->oic_conn->c_peer.nid));
+                }
+
                 imp->imp_conn_current = imp_conn;
         }
 
@@ -583,7 +587,7 @@ static int ptlrpc_first_transno(struct obd_import *imp, __u64 *transno)
  * actual sending.
  * Returns 0 on success or error code.
  */
-int ptlrpc_connect_import(struct obd_import *imp, char *new_uuid)
+int ptlrpc_connect_import(struct obd_import *imp)
 {
         struct obd_device *obd = imp->imp_obd;
         int initial_connect = 0;
@@ -628,15 +632,6 @@ int ptlrpc_connect_import(struct obd_import *imp, char *new_uuid)
                                            &imp->imp_connect_data.ocd_transno);
         cfs_spin_unlock(&imp->imp_lock);
 
-        if (new_uuid) {
-                struct obd_uuid uuid;
-
-                obd_str2uuid(&uuid, new_uuid);
-                rc = import_set_conn_priority(imp, &uuid);
-                if (rc)
-                        GOTO(out, rc);
-        }
-
         rc = import_select_connection(imp);
         if (rc)
                 GOTO(out, rc);
@@ -714,7 +709,7 @@ int ptlrpc_connect_import(struct obd_import *imp, char *new_uuid)
 
         DEBUG_REQ(D_RPCTRACE, request, "(re)connect request (timeout %d)",
                   request->rq_timeout);
-        ptlrpcd_add_req(request, PSCOPE_OTHER);
+        ptlrpcd_add_req(request, PDL_POLICY_ROUND, -1);
         rc = 0;
 out:
         if (rc != 0) {
@@ -728,8 +723,14 @@ EXPORT_SYMBOL(ptlrpc_connect_import);
 static void ptlrpc_maybe_ping_import_soon(struct obd_import *imp)
 {
 #ifdef __KERNEL__
-        /* the pinger takes care of issuing the next reconnect request */
-        return;
+        int force_verify;
+
+        cfs_spin_lock(&imp->imp_lock);
+        force_verify = imp->imp_force_verify != 0;
+        cfs_spin_unlock(&imp->imp_lock);
+
+        if (force_verify)
+                ptlrpc_pinger_wake_up();
 #else
         /* liblustre has no pinger thread, so we wakeup pinger anyway */
         ptlrpc_pinger_wake_up();
@@ -770,6 +771,7 @@ static int ptlrpc_connect_interpret(const struct lu_env *env,
                  * for connecting*/
                 imp->imp_force_reconnect = ptlrpc_busy_reconnect(rc);
                 cfs_spin_unlock(&imp->imp_lock);
+                ptlrpc_maybe_ping_import_soon(imp);
                 GOTO(out, rc);
         }
 
@@ -780,6 +782,7 @@ static int ptlrpc_connect_interpret(const struct lu_env *env,
         /* All imports are pingable */
         imp->imp_pingable = 1;
         imp->imp_force_reconnect = 0;
+        imp->imp_force_verify = 0;
 
         if (aa->pcaa_initial_connect) {
                 if (msg_flags & MSG_CONNECT_REPLAYABLE) {
@@ -820,10 +823,11 @@ static int ptlrpc_connect_interpret(const struct lu_env *env,
                 memset(&old_hdl, 0, sizeof(old_hdl));
                 if (!memcmp(&old_hdl, lustre_msg_get_handle(request->rq_repmsg),
                             sizeof (old_hdl))) {
-                        CERROR("%s@%s didn't like our handle "LPX64
-                               ", failed\n", obd2cli_tgt(imp->imp_obd),
-                               imp->imp_connection->c_remote_uuid.uuid,
-                               imp->imp_dlm_handle.cookie);
+                        LCONSOLE_WARN("Reconnect to %s (at @%s) failed due "
+                                      "bad handle "LPX64"\n",
+                                      obd2cli_tgt(imp->imp_obd),
+                                      imp->imp_connection->c_remote_uuid.uuid,
+                                      imp->imp_dlm_handle.cookie);
                         GOTO(out, rc = -ENOTCONN);
                 }
 
@@ -840,15 +844,27 @@ static int ptlrpc_connect_interpret(const struct lu_env *env,
                          * eviction. If it is in recovery - we are safe to
                          * participate since we can reestablish all of our state
                          * with server again */
-                        CDEBUG(level,"%s@%s changed server handle from "
-                                     LPX64" to "LPX64"%s\n",
-                                     obd2cli_tgt(imp->imp_obd),
-                                     imp->imp_connection->c_remote_uuid.uuid,
-                                     imp->imp_remote_handle.cookie,
-                                     lustre_msg_get_handle(request->rq_repmsg)->
-                                                                        cookie,
-                                     (MSG_CONNECT_RECOVERING & msg_flags) ?
-                                         " but is still in recovery" : "");
+                        if ((MSG_CONNECT_RECOVERING & msg_flags)) {
+                                CDEBUG(level,"%s@%s changed server handle from "
+                                       LPX64" to "LPX64
+                                       " but is still in recovery\n",
+                                       obd2cli_tgt(imp->imp_obd),
+                                       imp->imp_connection->c_remote_uuid.uuid,
+                                       imp->imp_remote_handle.cookie,
+                                       lustre_msg_get_handle(
+                                       request->rq_repmsg)->cookie);
+                        } else {
+                                LCONSOLE_WARN("Evicted from %s (at %s) "
+                                              "after server handle changed from "
+                                              LPX64" to "LPX64"\n",
+                                              obd2cli_tgt(imp->imp_obd),
+                                              imp->imp_connection-> \
+                                              c_remote_uuid.uuid,
+                                              imp->imp_remote_handle.cookie,
+                                              lustre_msg_get_handle(
+                                              request->rq_repmsg)->cookie);
+                        }
+
 
                         imp->imp_remote_handle =
                                      *lustre_msg_get_handle(request->rq_repmsg);
@@ -920,7 +936,7 @@ finish:
                                "invalidating and reconnecting\n",
                                obd2cli_tgt(imp->imp_obd),
                                imp->imp_connection->c_remote_uuid.uuid);
-                        ptlrpc_connect_import(imp, NULL);
+                        ptlrpc_connect_import(imp);
                         RETURN(0);
                 }
         } else {
@@ -948,6 +964,8 @@ finish:
                 }
 
                 imp->imp_connect_data = *ocd;
+                CDEBUG(D_HA, "obd %s to target with inst %u\n",
+                       imp->imp_obd->obd_name, ocd->ocd_instance);
 
                 exp = class_conn2export(&imp->imp_dlm_handle);
                 cfs_spin_unlock(&imp->imp_lock);
@@ -1155,7 +1173,7 @@ static int completed_replay_interpret(const struct lu_env *env,
                                req->rq_import->imp_obd->obd_name,
                                req->rq_status);
                 }
-                ptlrpc_connect_import(req->rq_import, NULL);
+                ptlrpc_connect_import(req->rq_import);
         }
 
         RETURN(0);
@@ -1191,7 +1209,7 @@ static int signal_completed_replay(struct obd_import *imp)
                 req->rq_timeout *= 3;
         req->rq_interpret_reply = completed_replay_interpret;
 
-        ptlrpcd_add_req(req, PSCOPE_OTHER);
+        ptlrpcd_add_req(req, PDL_POLICY_ROUND, -1);
         RETURN(0);
 }
 
@@ -1340,8 +1358,8 @@ int ptlrpc_import_recovery_state_machine(struct obd_import *imp)
 
                 deuuidify(obd2cli_tgt(imp->imp_obd), NULL,
                           &target_start, &target_len);
-                LCONSOLE_INFO("%s: Connection restored to service %.*s "
-                              "using nid %s.\n", imp->imp_obd->obd_name,
+                LCONSOLE_INFO("%s: Connection restored to %.*s (at %s)\n",
+                              imp->imp_obd->obd_name,
                               target_len, target_start,
                               libcfs_nid2str(imp->imp_connection->c_peer.nid));
         }