Whamcloud - gitweb
b=6063
[fs/lustre-release.git] / lustre / ptlrpc / import.c
index 5bc9e3f..52f3587 100644 (file)
  */
 
 #define DEBUG_SUBSYSTEM S_RPC
-#ifdef __KERNEL__
-# include <linux/config.h>
-# include <linux/module.h>
-# include <linux/kmod.h>
-#else
+#ifndef __KERNEL__
 # include <liblustre.h>
 #endif
 
 #include <linux/lustre_export.h>
 #include <linux/obd.h>
 #include <linux/obd_class.h>
+#include <linux/lustre_sec.h>
 
 #include "ptlrpc_internal.h"
 
 struct ptlrpc_connect_async_args {
          __u64 pcaa_peer_committed;
         int pcaa_initial_connect;
-        int pcaa_was_invalid;
 };
 
 /* A CLOSED import should remain so. */
@@ -79,7 +75,7 @@ int ptlrpc_import_recovery_state_machine(struct obd_import *imp);
 int ptlrpc_init_import(struct obd_import *imp)
 {
         unsigned long flags;
-        
+
         spin_lock_irqsave(&imp->imp_lock, flags);
 
         imp->imp_generation++;
@@ -97,48 +93,234 @@ int ptlrpc_set_import_discon(struct obd_import *imp)
 {
         unsigned long flags;
         int rc = 0;
-        
+
         spin_lock_irqsave(&imp->imp_lock, flags);
 
         if (imp->imp_state == LUSTRE_IMP_FULL) {
+                CWARN("%s: connection lost to %s@%s\n",
+                      imp->imp_obd->obd_name, 
+                      imp->imp_target_uuid.uuid,
+                      imp->imp_connection->c_remote_uuid.uuid);
+                ptlrpc_deactivate_timeouts();
                 IMPORT_SET_STATE_NOLOCK(imp, LUSTRE_IMP_DISCON);
+                spin_unlock_irqrestore(&imp->imp_lock, flags);
+                obd_import_event(imp->imp_obd, imp, IMP_EVENT_DISCON);
                 rc = 1;
         } else {
+                spin_unlock_irqrestore(&imp->imp_lock, flags);
                 CDEBUG(D_HA, "%p %s: import already not connected: %s\n",
-                       imp,imp->imp_client->cli_name, 
+                       imp,imp->imp_client->cli_name,
                        ptlrpc_import_state_name(imp->imp_state));
         }
-        spin_unlock_irqrestore(&imp->imp_lock, flags);
 
         return rc;
 }
 
+/*
+ * This acts as a barrier; all existing requests are rejected, and
+ * no new requests will be accepted until the import is valid again.
+ */
+void ptlrpc_deactivate_import(struct obd_import *imp)
+{
+        unsigned long flags;
+        ENTRY;
+
+        spin_lock_irqsave(&imp->imp_lock, flags);
+        CDEBUG(D_HA, "setting import %s INVALID\n",
+               imp->imp_target_uuid.uuid);
+        imp->imp_invalid = 1;
+        imp->imp_generation++;
+        spin_unlock_irqrestore(&imp->imp_lock, flags);
+
+        ptlrpc_abort_inflight(imp);
+        obd_import_event(imp->imp_obd, imp, IMP_EVENT_INACTIVE);
+}
+
+/*
+ * This function will invalidate the import, if necessary, then block
+ * for all the RPC completions, and finally notify the obd to
+ * invalidate its state (ie cancel locks, clear pending requests,
+ * etc).
+ *
+ * in_rpc: true if this is called while processing an rpc, like
+ *    CONNECT. It will allow for one RPC to be inflight while
+ *    waiting for requests to complete. Ugly, yes, but I don't see an
+ *    cleaner way right now.
+ */
+void ptlrpc_invalidate_import(struct obd_import *imp, int in_rpc)
+{
+        struct l_wait_info lwi;
+        unsigned long timeout;
+        int inflight = 0;
+        int rc;
+
+        if (!imp->imp_invalid)
+                ptlrpc_deactivate_import(imp);
+
+        LASSERT(imp->imp_invalid);
+
+        if (in_rpc)
+                inflight = 1;
+
+        /* wait for all requests to error out and call completion 
+           callbacks */
+        if (imp->imp_server_timeout)
+                timeout = obd_timeout / 2;
+        else
+                timeout = obd_timeout;
+        timeout = MAX(timeout * HZ, 1);
+        lwi = LWI_TIMEOUT_INTR(timeout, NULL, NULL, NULL);
+        rc = l_wait_event(imp->imp_recovery_waitq, 
+                          (atomic_read(&imp->imp_inflight) == inflight), 
+                          &lwi);
+
+        if (rc)
+                CERROR("%s: rc = %d waiting for callback (%d != %d)\n",
+                       imp->imp_target_uuid.uuid, rc,
+                       atomic_read(&imp->imp_inflight), !!in_rpc);
+
+        obd_import_event(imp->imp_obd, imp, IMP_EVENT_INVALIDATE);
+}
+
+void ptlrpc_activate_import(struct obd_import *imp)
+{
+        struct obd_device *obd = imp->imp_obd;
+        unsigned long flags;
+
+        spin_lock_irqsave(&imp->imp_lock, flags);
+        imp->imp_invalid = 0;
+        spin_unlock_irqrestore(&imp->imp_lock, flags);
+
+        obd_import_event(obd, imp, IMP_EVENT_ACTIVE);
+        ptlrpc_activate_timeouts();
+}
+
 void ptlrpc_fail_import(struct obd_import *imp, int generation)
 {
         ENTRY;
 
         LASSERT (!imp->imp_dlm_fake);
 
-        if (ptlrpc_set_import_discon(imp))
-                ptlrpc_handle_failed_import(imp);
+        if (ptlrpc_set_import_discon(imp)) {
+                unsigned long flags;
+
+                if (!imp->imp_replayable) {
+                        CDEBUG(D_HA, "import %s@%s for %s not replayable, "
+                               "auto-deactivating\n",
+                               imp->imp_target_uuid.uuid,
+                               imp->imp_connection->c_remote_uuid.uuid,
+                               imp->imp_obd->obd_name);
+                        ptlrpc_deactivate_import(imp);
+                }
+
+                CDEBUG(D_HA, "%s: waking up pinger\n",
+                       imp->imp_target_uuid.uuid);
 
+                spin_lock_irqsave(&imp->imp_lock, flags);
+                imp->imp_force_verify = 1;
+                spin_unlock_irqrestore(&imp->imp_lock, flags);
+
+                ptlrpc_pinger_wake_up();
+        }
         EXIT;
 }
 
+#define ATTEMPT_TOO_SOON(last)  \
+        ((last) && ((long)(jiffies - (last)) <= (long)(obd_timeout * 2 * HZ)))
+
+static int import_select_connection(struct obd_import *imp)
+{
+        struct obd_import_conn *imp_conn, *tmp;
+        struct obd_export *dlmexp;
+        int found = 0;
+        ENTRY;
+
+        spin_lock(&imp->imp_lock);
+
+        if (list_empty(&imp->imp_conn_list)) {
+                CERROR("no available connections on imp %p@%s\n",
+                        imp, imp->imp_obd->obd_name);
+                spin_unlock(&imp->imp_lock);
+                RETURN(-EINVAL);
+        }
+
+        list_for_each_entry(imp_conn, &imp->imp_conn_list, oic_item) {
+                if (!ATTEMPT_TOO_SOON(imp_conn->oic_last_attempt)) {
+                        found = 1;
+                        break;
+                }
+        }
+
+        /* if not found, simply choose the current one */
+        if (!found) {
+                CWARN("obd %s imp 0x%p: all connections have been "
+                      "tried recently\n", imp->imp_obd->obd_name, imp);
+                LASSERT(imp->imp_conn_current);
+                imp_conn = imp->imp_conn_current;
+        }
+        LASSERT(imp_conn->oic_conn);
+
+        imp_conn->oic_last_attempt = jiffies;
+
+        /* move the items ahead of the selected one to list tail */
+        while (1) {
+                tmp= list_entry(imp->imp_conn_list.next,
+                                struct obd_import_conn, oic_item);
+                if (tmp == imp_conn)
+                        break;
+                list_del(&tmp->oic_item);
+                list_add_tail(&tmp->oic_item, &imp->imp_conn_list);
+        }
+
+        /* switch connection if we chose a new one */
+        if (imp->imp_connection != imp_conn->oic_conn) {
+                if (imp->imp_connection) {
+                        ptlrpcs_sec_invalidate_cache(imp->imp_sec);
+                        ptlrpc_put_connection(imp->imp_connection);
+                }
+                imp->imp_connection =
+                        ptlrpc_connection_addref(imp_conn->oic_conn);
+        }
+
+        dlmexp =  class_conn2export(&imp->imp_dlm_handle);
+        LASSERT(dlmexp != NULL);
+        if (dlmexp->exp_connection)
+                ptlrpc_put_connection(imp->imp_connection);
+        dlmexp->exp_connection = ptlrpc_connection_addref(imp_conn->oic_conn);
+        class_export_put(dlmexp);
+
+        imp->imp_conn_current = imp_conn;
+        CWARN("obd %s imp 0x%p: select conn %s\n",
+               imp->imp_obd->obd_name, imp,
+               imp_conn->oic_uuid.uuid);
+        spin_unlock(&imp->imp_lock);
+
+        RETURN(0);
+}
+
+
+
 int ptlrpc_connect_import(struct obd_import *imp, char * new_uuid)
 {
         struct obd_device *obd = imp->imp_obd;
         int initial_connect = 0;
         int rc;
         __u64 committed_before_reconnect = 0;
-        int was_invalid = 0;
         struct ptlrpc_request *request;
-        int size[] = {sizeof(imp->imp_target_uuid),
-                                 sizeof(obd->obd_uuid),
-                                 sizeof(imp->imp_dlm_handle)};
-        char *tmp[] = {imp->imp_target_uuid.uuid,
+        int size[] = {0,
+                      sizeof(imp->imp_target_uuid),
+                      sizeof(obd->obd_uuid),
+                      sizeof(imp->imp_dlm_handle),
+                      sizeof(imp->imp_connect_flags),
+                      sizeof(imp->imp_connect_data)};
+        char *tmp[] = {NULL,
+                       imp->imp_target_uuid.uuid,
                        obd->obd_uuid.uuid,
-                       (char *)&imp->imp_dlm_handle};
+                       (char *)&imp->imp_dlm_handle,
+                       (char *)&imp->imp_connect_flags, /* XXX: make this portable! */
+                       (char*) &imp->imp_connect_data};
+        int repsize = sizeof(struct obd_connect_data);
+                        
         struct ptlrpc_connect_async_args *aa;
         unsigned long flags;
 
@@ -159,68 +341,50 @@ int ptlrpc_connect_import(struct obd_import *imp, char * new_uuid)
 
         IMPORT_SET_STATE_NOLOCK(imp, LUSTRE_IMP_CONNECTING);
 
-        imp->imp_conn_cnt++; 
-        imp->imp_last_replay_transno = 0;
+        imp->imp_resend_replay = 0;
 
         if (imp->imp_remote_handle.cookie == 0) {
                 initial_connect = 1;
         } else {
                 committed_before_reconnect = imp->imp_peer_committed_transno;;
-
+                imp->imp_conn_cnt++;
         }
 
-        if (imp->imp_invalid) {
-                imp->imp_invalid = 0;
-                was_invalid = 1;
-        }
 
         spin_unlock_irqrestore(&imp->imp_lock, flags);
 
         if (new_uuid) {
-                struct ptlrpc_connection *conn;
                 struct obd_uuid uuid;
-                struct obd_export *dlmexp;
 
                 obd_str2uuid(&uuid, new_uuid);
 
-                conn = ptlrpc_uuid_to_connection(&uuid);
-                if (!conn)
-                        GOTO(out, rc = -ENOENT);
-
-                CDEBUG(D_HA, "switching import %s/%s from %s to %s\n",
-                       imp->imp_target_uuid.uuid, imp->imp_obd->obd_name,
-                       imp->imp_connection->c_remote_uuid.uuid,
-                       conn->c_remote_uuid.uuid);
-
-                /* Switch the import's connection and the DLM export's
-                 * connection (which are almost certainly the same, but we
-                 * keep distinct refs just to make things clearer. I think. */
-                if (imp->imp_connection)
-                        ptlrpc_put_connection(imp->imp_connection);
-                /* We hand off the ref from ptlrpc_get_connection. */
-                imp->imp_connection = conn;
-
-                dlmexp = class_conn2export(&imp->imp_dlm_handle);
-                
-                LASSERT(dlmexp != NULL);
-
-                if (dlmexp->exp_connection)
-                        ptlrpc_put_connection(dlmexp->exp_connection);
-                dlmexp->exp_connection = ptlrpc_connection_addref(conn);
-                class_export_put(dlmexp);
-
+                rc = import_set_conn_priority(imp, &uuid);
+                if (rc)
+                        GOTO(out, rc);
         }
+        rc = import_select_connection(imp);
+        if (rc)
+                GOTO(out, rc);
+
+        LASSERT(imp->imp_sec);
 
-        request = ptlrpc_prep_req(imp, imp->imp_connect_op, 3, size, tmp);
+        size[0] = lustre_secdesc_size();
+        request = ptlrpc_prep_req(imp, LUSTRE_OBD_VERSION,
+                                  imp->imp_connect_op, 6, size, tmp);
         if (!request)
                 GOTO(out, rc = -ENOMEM);
 
+        lustre_pack_secdesc(request, size[0]);
+
 #ifndef __KERNEL__
         lustre_msg_add_op_flags(request->rq_reqmsg, MSG_CONNECT_LIBCLIENT);
 #endif
+        if (obd->u.cli.cl_async) {
+                lustre_msg_add_op_flags(request->rq_reqmsg, MSG_CONNECT_ASYNC);
+        }
 
         request->rq_send_state = LUSTRE_IMP_CONNECTING;
-        request->rq_replen = lustre_msg_size(0, NULL);
+        request->rq_replen = lustre_msg_size(1, &repsize);
         request->rq_interpret_reply = ptlrpc_connect_interpret;
 
         LASSERT (sizeof (*aa) <= sizeof (request->rq_async_args));
@@ -229,13 +393,18 @@ int ptlrpc_connect_import(struct obd_import *imp, char * new_uuid)
 
         aa->pcaa_peer_committed = committed_before_reconnect;
         aa->pcaa_initial_connect = initial_connect;
-        aa->pcaa_was_invalid = was_invalid;
 
-        if (aa->pcaa_initial_connect)
-                imp->imp_replayable = 1;
+        if (aa->pcaa_initial_connect) {
+                lustre_msg_add_op_flags(request->rq_reqmsg, 
+                                        MSG_CONNECT_INITIAL);
+                imp->imp_replayable = 1; 
+        }
+        
+        imp->imp_reqs_replayed = imp->imp_locks_replayed = 0;
 
         ptlrpcd_add_req(request);
         rc = 0;
+        imp->imp_connect_start = jiffies;
 out:
         if (rc != 0) {
                 IMPORT_SET_STATE(imp, LUSTRE_IMP_DISCON);
@@ -245,7 +414,7 @@ out:
 }
 
 static int ptlrpc_connect_interpret(struct ptlrpc_request *request,
-                                    void * data, int rc)
+                                    void *data, int rc)
 {
         struct ptlrpc_connect_async_args *aa = data;
         struct obd_import *imp = request->rq_import;
@@ -253,7 +422,7 @@ static int ptlrpc_connect_interpret(struct ptlrpc_request *request,
         unsigned long flags;
         int msg_flags;
         ENTRY;
-        
+
         spin_lock_irqsave(&imp->imp_lock, flags);
         if (imp->imp_state == LUSTRE_IMP_CLOSED) {
                 spin_unlock_irqrestore(&imp->imp_lock, flags);
@@ -263,20 +432,38 @@ static int ptlrpc_connect_interpret(struct ptlrpc_request *request,
 
         if (rc)
                 GOTO(out, rc);
-
+        LASSERT(imp->imp_conn_current);
+        imp->imp_conn_current->oic_last_attempt = 0;
+/*
+        remote_flag = lustre_msg_buf(request->rq_repmsg, 0, sizeof(int));
+        LASSERT(remote_flag != NULL);
+        imp->imp_obd->u.cli.cl_remote = *remote_flag;
+*/
         msg_flags = lustre_msg_get_op_flags(request->rq_repmsg);
 
         if (aa->pcaa_initial_connect) {
+                struct obd_connect_data *conn_data;
+
+                conn_data = lustre_swab_repbuf(request, 0, sizeof(*conn_data),
+                                               lustre_swab_connect);
+                LASSERT(conn_data);
+                imp->imp_connect_data.ocd_connect_flags =
+                                        conn_data->ocd_connect_flags;
+
                 if (msg_flags & MSG_CONNECT_REPLAYABLE) {
                         CDEBUG(D_HA, "connected to replayable target: %s\n",
                                imp->imp_target_uuid.uuid);
-                        imp->imp_replayable = 1;
-                        ptlrpc_pinger_add_import(imp);
+                        imp->imp_pingable = imp->imp_replayable = 1;
                 } else {
                         imp->imp_replayable = 0;
                 }
+                LASSERTF(imp->imp_conn_cnt < request->rq_repmsg->conn_cnt,
+                         "imp conn_cnt %d req conn_cnt %d", 
+                         imp->imp_conn_cnt, request->rq_repmsg->conn_cnt);
+                imp->imp_conn_cnt = request->rq_repmsg->conn_cnt;
                 imp->imp_remote_handle = request->rq_repmsg->handle;
                 IMPORT_SET_STATE(imp, LUSTRE_IMP_FULL);
+                ptlrpc_pinger_sending_on_import(imp);
                 GOTO(finish, rc = 0);
         }
 
@@ -302,26 +489,35 @@ static int ptlrpc_connect_interpret(struct ptlrpc_request *request,
                                request->rq_repmsg->handle.cookie);
                         imp->imp_remote_handle = request->rq_repmsg->handle;
                 } else {
-                        CERROR("reconnected to %s@%s after partition\n",
-                               imp->imp_target_uuid.uuid, 
+                        CDEBUG(D_HA, "reconnected to %s@%s after partition\n",
+                               imp->imp_target_uuid.uuid,
                                imp->imp_connection->c_remote_uuid.uuid);
                 }
-                IMPORT_SET_STATE(imp, LUSTRE_IMP_RECOVER);
-        } 
-        else if (MSG_CONNECT_RECOVERING & msg_flags) {
+
+                if (imp->imp_invalid) {
+                        IMPORT_SET_STATE(imp, LUSTRE_IMP_EVICTED);
+                } else if (MSG_CONNECT_RECOVERING & msg_flags) {
+                        CDEBUG(D_HA, "%s: reconnected to %s during replay\n",
+                               imp->imp_obd->obd_name, 
+                               imp->imp_target_uuid.uuid);
+                        imp->imp_resend_replay = 1;
+                        IMPORT_SET_STATE(imp, LUSTRE_IMP_REPLAY);
+                } else {
+                        IMPORT_SET_STATE(imp, LUSTRE_IMP_RECOVER);
+                }
+        } else if ((MSG_CONNECT_RECOVERING & msg_flags) && !imp->imp_invalid) {
                 LASSERT(imp->imp_replayable);
-                imp->imp_state = LUSTRE_IMP_RECOVER;
                 imp->imp_remote_handle = request->rq_repmsg->handle;
+                imp->imp_last_replay_transno = 0;
                 IMPORT_SET_STATE(imp, LUSTRE_IMP_REPLAY);
-        } 
-        else {
+        } else {
+                CDEBUG(D_HA, "oops! we get evicted from %s\n", imp->imp_target_uuid.uuid);
                 imp->imp_remote_handle = request->rq_repmsg->handle;
                 IMPORT_SET_STATE(imp, LUSTRE_IMP_EVICTED);
         }
-        
+
         /* Sanity checks for a reconnected import. */
-        if (!(imp->imp_replayable) != 
-             !(msg_flags & MSG_CONNECT_REPLAYABLE)) {
+        if (!(imp->imp_replayable) != !(msg_flags & MSG_CONNECT_REPLAYABLE)) {
                 CERROR("imp_replayable flag does not match server "
                        "after reconnect. We should LBUG right here.\n");
         }
@@ -338,10 +534,6 @@ static int ptlrpc_connect_interpret(struct ptlrpc_request *request,
 finish:
         rc = ptlrpc_import_recovery_state_machine(imp);
         if (rc != 0) {
-                if (aa->pcaa_was_invalid) {
-                        ptlrpc_set_import_active(imp, 0);
-                }                
-
                 if (rc == -ENOTCONN) {
                         CDEBUG(D_HA, "evicted/aborted by %s@%s during recovery;"
                                "invalidating and reconnecting\n",
@@ -349,32 +541,35 @@ finish:
                                imp->imp_connection->c_remote_uuid.uuid);
                         ptlrpc_connect_import(imp, NULL);
                         RETURN(0);
-                } 
+                }
         }
  out:
         if (rc != 0) {
                 IMPORT_SET_STATE(imp, LUSTRE_IMP_DISCON);
-                if (aa->pcaa_initial_connect && !imp->imp_initial_recov) {
-                        ptlrpc_set_import_active(imp, 0);
-                        GOTO(norecov, rc);
-                }
-                CDEBUG(D_ERROR, 
-                       "recovery of %s on %s failed (%d); restarting\n",
+                if (aa->pcaa_initial_connect && !imp->imp_initial_recov)
+                        ptlrpc_deactivate_import(imp);
+                CDEBUG(D_HA, "recovery of %s on %s failed (%d)\n",
                        imp->imp_target_uuid.uuid,
                        (char *)imp->imp_connection->c_remote_uuid.uuid, rc);
-                ptlrpc_handle_failed_import(imp);
         }
 
-norecov:
         wake_up(&imp->imp_recovery_waitq);
         RETURN(rc);
 }
 
 static int completed_replay_interpret(struct ptlrpc_request *req,
-                                    void * data, int rc)
+                                      void *data, int rc)
 {
         atomic_dec(&req->rq_import->imp_replay_inflight);
-        ptlrpc_import_recovery_state_machine(req->rq_import);
+        if (req->rq_status == 0) {
+                ptlrpc_import_recovery_state_machine(req->rq_import);
+        } else {
+                CDEBUG(D_HA, "%s: LAST_REPLAY message error: %d, "
+                       "reconnecting\n", 
+                       req->rq_import->imp_obd->obd_name, req->rq_status);
+                ptlrpc_connect_import(req->rq_import, NULL);
+        }
+
         RETURN(0);
 }
 
@@ -386,38 +581,82 @@ static int signal_completed_replay(struct obd_import *imp)
         LASSERT(atomic_read(&imp->imp_replay_inflight) == 0);
         atomic_inc(&imp->imp_replay_inflight);
 
-        req = ptlrpc_prep_req(imp, OBD_PING, 0, NULL, NULL);
-        if (!req)
+        req = ptlrpc_prep_req(imp, LUSTRE_OBD_VERSION, OBD_PING, 0, NULL, NULL);
+        if (!req) {
+                atomic_dec(&imp->imp_replay_inflight);
                 RETURN(-ENOMEM);
+        }
 
         req->rq_replen = lustre_msg_size(0, NULL);
         req->rq_send_state = LUSTRE_IMP_REPLAY_WAIT;
-        req->rq_reqmsg->flags |= MSG_LAST_REPLAY;
-        req->rq_timeout *= 3; 
+        req->rq_reqmsg->flags |= MSG_LOCK_REPLAY_DONE | MSG_REQ_REPLAY_DONE;
+        req->rq_timeout *= 3;
         req->rq_interpret_reply = completed_replay_interpret;
 
         ptlrpcd_add_req(req);
         RETURN(0);
 }
 
+#ifdef __KERNEL__
+static int ptlrpc_invalidate_import_thread(void *data)
+{
+        struct obd_import *imp = data;
+        unsigned long flags;
+
+        ENTRY;
+
+        lock_kernel();
+        ptlrpc_daemonize();
+
+        SIGNAL_MASK_LOCK(current, flags);
+        sigfillset(&current->blocked);
+        RECALC_SIGPENDING;
+        SIGNAL_MASK_UNLOCK(current, flags);
+        THREAD_NAME(current->comm, sizeof(current->comm), "ll_imp_inval");
+        unlock_kernel();
+
+        CDEBUG(D_HA, "thread invalidate import %s to %s@%s\n",
+               imp->imp_obd->obd_name, imp->imp_target_uuid.uuid,
+               imp->imp_connection->c_remote_uuid.uuid);
+
+        ptlrpc_invalidate_import(imp, 0);
+        IMPORT_SET_STATE(imp, LUSTRE_IMP_RECOVER);
+
+        ptlrpc_import_recovery_state_machine(imp);
+
+        RETURN(0);
+}
+#endif
 
 int ptlrpc_import_recovery_state_machine(struct obd_import *imp)
 {
         int rc = 0;
+        int inflight;
 
         if (imp->imp_state == LUSTRE_IMP_EVICTED) {
                 CDEBUG(D_HA, "evicted from %s@%s; invalidating\n",
                        imp->imp_target_uuid.uuid,
                        imp->imp_connection->c_remote_uuid.uuid);
-                ptlrpc_set_import_active(imp, 0);
+
+#ifdef __KERNEL__
+                rc = kernel_thread(ptlrpc_invalidate_import_thread, imp,
+                                   CLONE_VM | CLONE_FILES);
+                if (rc < 0)
+                        CERROR("error starting invalidate thread: %d\n", rc);
+                RETURN(rc);
+#else
+                ptlrpc_invalidate_import(imp, 1);
+
                 IMPORT_SET_STATE(imp, LUSTRE_IMP_RECOVER);
-        } 
-        
+#endif
+        }
+
         if (imp->imp_state == LUSTRE_IMP_REPLAY) {
                 CDEBUG(D_HA, "replay requested by %s\n",
                        imp->imp_target_uuid.uuid);
-                rc = ptlrpc_replay_next(imp);
-                if (rc == 0 && atomic_read(&imp->imp_replay_inflight) == 0) {
+                rc = ptlrpc_replay_next(imp, &inflight);
+                if (inflight == 0 &&
+                    atomic_read(&imp->imp_replay_inflight) == 0) {
                         IMPORT_SET_STATE(imp, LUSTRE_IMP_REPLAY_LOCKS);
                         rc = ldlm_replay_locks(imp);
                         if (rc)
@@ -447,12 +686,19 @@ int ptlrpc_import_recovery_state_machine(struct obd_import *imp)
                        imp->imp_target_uuid.uuid,
                        imp->imp_connection->c_remote_uuid.uuid);
 
-                ptlrpc_set_import_active(imp, 1);
                 rc = ptlrpc_resend(imp);
                 if (rc)
                         GOTO(out, rc);
                 IMPORT_SET_STATE(imp, LUSTRE_IMP_FULL);
-        } 
+                ptlrpc_activate_import(imp);
+                CWARN("%s: connection restored to %s@%s, "
+                       "%d/%d req/lock replayed\n",
+                      imp->imp_obd->obd_name, 
+                      imp->imp_target_uuid.uuid,
+                      imp->imp_connection->c_remote_uuid.uuid,
+                      imp->imp_reqs_replayed,
+                      imp->imp_locks_replayed);
+        }
 
         if (imp->imp_state == LUSTRE_IMP_FULL) {
                 wake_up(&imp->imp_recovery_waitq);
@@ -463,7 +709,7 @@ int ptlrpc_import_recovery_state_machine(struct obd_import *imp)
         RETURN(rc);
 }
 
-static int back_to_sleep(void *unused) 
+static int back_to_sleep(void *unused)
 {
        return 0;
 }
@@ -479,7 +725,7 @@ int ptlrpc_disconnect_import(struct obd_import *imp)
         switch (imp->imp_connect_op) {
         case OST_CONNECT: rq_opc = OST_DISCONNECT; break;
         case MDS_CONNECT: rq_opc = MDS_DISCONNECT; break;
-        case MGMT_CONNECT:rq_opc = MGMT_DISCONNECT;break;
+        case MGMT_CONNECT: rq_opc = MGMT_DISCONNECT; break;
         default:
                 CERROR("don't know how to disconnect from %s (connect_op %d)\n",
                        imp->imp_target_uuid.uuid, imp->imp_connect_op);
@@ -489,8 +735,13 @@ int ptlrpc_disconnect_import(struct obd_import *imp)
 
         if (ptlrpc_import_in_recovery(imp)) {
                 struct l_wait_info lwi;
-                lwi = LWI_TIMEOUT_INTR(MAX(obd_timeout * HZ, 1), back_to_sleep, 
-                                       NULL, NULL);
+                unsigned long timeout;
+                if (imp->imp_server_timeout)
+                        timeout = obd_timeout / 2;
+                else
+                        timeout = obd_timeout;
+                timeout = MAX(timeout * HZ, 1);
+                lwi = LWI_TIMEOUT_INTR(obd_timeout, back_to_sleep, NULL, NULL);
                 rc = l_wait_event(imp->imp_recovery_waitq, 
                                   !ptlrpc_import_in_recovery(imp), &lwi);
 
@@ -502,13 +753,15 @@ int ptlrpc_disconnect_import(struct obd_import *imp)
         }
         spin_unlock_irqrestore(&imp->imp_lock, flags);
 
-        request = ptlrpc_prep_req(imp, rq_opc, 0, NULL, NULL);
+        request = ptlrpc_prep_req(imp, LUSTRE_OBD_VERSION, rq_opc,
+                                  0, NULL, NULL);
         if (request) {
                 /* For non-replayable connections, don't attempt
                    reconnect if this fails */
                 if (!imp->imp_replayable) {
-                        IMPORT_SET_STATE(imp, LUSTRE_IMP_DISCON);
-                        request->rq_send_state =  LUSTRE_IMP_DISCON;
+                        request->rq_no_resend = 1;
+                        IMPORT_SET_STATE(imp, LUSTRE_IMP_CONNECTING);
+                        request->rq_send_state =  LUSTRE_IMP_CONNECTING;
                 }
                 request->rq_replen = lustre_msg_size(0, NULL);
                 rc = ptlrpc_queue_wait(request);
@@ -519,6 +772,7 @@ int ptlrpc_disconnect_import(struct obd_import *imp)
 out:
         IMPORT_SET_STATE_NOLOCK(imp, LUSTRE_IMP_CLOSED);
         memset(&imp->imp_remote_handle, 0, sizeof(imp->imp_remote_handle));
+        imp->imp_conn_cnt = 0;
         spin_unlock_irqrestore(&imp->imp_lock, flags);
 
         RETURN(rc);