Whamcloud - gitweb
LU-9274 ptlrpc: add replay request into unreplied list
[fs/lustre-release.git] / lustre / ptlrpc / recover.c
index e51e346..c526e9e 100644 (file)
-/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
- * vim:expandtab:shiftwidth=8:tabstop=8:
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
  *
- * Portal-RPC reconnection and replay operations, for use in recovery.
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
  *
- *  Copyright (c) 2002, 2003 Cluster File Systems, Inc.
- *   Author: Mike Shaver <shaver@clusterfs.com>
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html
  *
- *   This file is part of Lustre, http://www.lustre.org.
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Use is subject to license terms.
  *
- *   Lustre is free software; you can redistribute it and/or
- *   modify it under the terms of version 2 of the GNU General Public
- *   License as published by the Free Software Foundation.
+ * Copyright (c) 2011, 2016, Intel Corporation.
+ */
+/*
+ * This file is part of Lustre, http://www.lustre.org/
+ * Lustre is a trademark of Sun Microsystems, Inc.
  *
- *   Lustre is distributed in the hope that it will be useful,
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *   GNU General Public License for more details.
+ * lustre/ptlrpc/recover.c
  *
- *   You should have received a copy of the GNU General Public License
- *   along with Lustre; if not, write to the Free Software
- *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ * Author: Mike Shaver <shaver@clusterfs.com>
  */
 
 #define DEBUG_SUBSYSTEM S_RPC
-#ifdef __KERNEL__
-# include <linux/config.h>
-# include <linux/module.h>
-# include <linux/kmod.h>
-#else
-# include <liblustre.h>
-#endif
-
-#include <linux/obd_support.h>
-#include <linux/lustre_ha.h>
-#include <linux/lustre_net.h>
-#include <linux/lustre_import.h>
-#include <linux/lustre_export.h>
-#include <linux/obd.h>
-#include <linux/obd_ost.h>
-#include <linux/obd_class.h>
-#include <linux/obd_lov.h> /* for IOC_LOV_SET_OSC_ACTIVE */
+#include <linux/list.h>
+#include <libcfs/libcfs.h>
+#include <obd_support.h>
+#include <lustre_ha.h>
+#include <lustre_net.h>
+#include <lustre_import.h>
+#include <lustre_export.h>
+#include <obd.h>
+#include <obd_class.h>
 
 #include "ptlrpc_internal.h"
 
-static int ptlrpc_recover_import_no_retry(struct obd_import *, char *);
-
-void ptlrpc_run_recovery_over_upcall(struct obd_device *obd)
-{
-        char *argv[4];
-        char *envp[3];
-        int rc;
-        ENTRY;
-
-        argv[0] = obd_lustre_upcall;
-        argv[1] = "RECOVERY_OVER";
-        argv[2] = obd->obd_uuid.uuid;
-        argv[3] = NULL;
-        
-        envp[0] = "HOME=/";
-        envp[1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
-        envp[2] = NULL;
-
-        rc = USERMODEHELPER(argv[0], argv, envp);
-        if (rc < 0) {
-                CERROR("Error invoking recovery upcall %s %s %s: %d; check "
-                       "/proc/sys/lustre/upcall\n",
-                       argv[0], argv[1], argv[2], rc);
-
-        } else {
-                CWARN("Invoked upcall %s %s %s\n",
-                      argv[0], argv[1], argv[2]);
-        }
-}
-
-void ptlrpc_run_failed_import_upcall(struct obd_import* imp)
-{
-#ifdef __KERNEL__
-        unsigned long flags;
-        char *argv[7];
-        char *envp[3];
-        int rc;
-        ENTRY;
-
-        spin_lock_irqsave(&imp->imp_lock, flags);
-        if (imp->imp_state == LUSTRE_IMP_CLOSED) {
-                spin_unlock_irqrestore(&imp->imp_lock, flags);
-                EXIT;
-                return;
-        }
-        spin_unlock_irqrestore(&imp->imp_lock, flags);
-        
-        argv[0] = obd_lustre_upcall;
-        argv[1] = "FAILED_IMPORT";
-        argv[2] = imp->imp_target_uuid.uuid;
-        argv[3] = imp->imp_obd->obd_name;
-        argv[4] = imp->imp_connection->c_remote_uuid.uuid;
-        argv[5] = imp->imp_obd->obd_uuid.uuid;
-        argv[6] = NULL;
-
-        envp[0] = "HOME=/";
-        envp[1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
-        envp[2] = NULL;
-
-        rc = USERMODEHELPER(argv[0], argv, envp);
-        if (rc < 0) {
-                CERROR("Error invoking recovery upcall %s %s %s %s %s: %d; "
-                       "check /proc/sys/lustre/upcall\n",
-                       argv[0], argv[1], argv[2], argv[3], argv[4],rc);
-
-        } else {
-                CWARN("Invoked upcall %s %s %s %s %s\n",
-                      argv[0], argv[1], argv[2], argv[3], argv[4]);
-        }
-#else
-        if (imp->imp_state == LUSTRE_IMP_CLOSED) {
-                EXIT;
-                return;
-        }
-        ptlrpc_recover_import(imp, NULL);
-#endif
-}
-
-/* This might block waiting for the upcall to start, so it should
- * not be called from a thread that shouldn't block. (Like ptlrpcd) */
+/**
+ * Start recovery on disconnected import.
+ * This is done by just attempting a connect
+ */
 void ptlrpc_initiate_recovery(struct obd_import *imp)
 {
         ENTRY;
 
-        LASSERT (obd_lustre_upcall != NULL);
-        
-        if (strcmp(obd_lustre_upcall, "DEFAULT") == 0) {
-                CDEBUG(D_HA, "%s: starting recovery without upcall\n",
-                        imp->imp_target_uuid.uuid);
-                ptlrpc_connect_import(imp, NULL);
-        } 
-        else if (strcmp(obd_lustre_upcall, "NONE") == 0) {
-                CDEBUG(D_HA, "%s: recovery disabled\n",
-                        imp->imp_target_uuid.uuid);
-        } 
-        else {
-                CDEBUG(D_HA, "%s: calling upcall to start recovery\n",
-                        imp->imp_target_uuid.uuid);
-                ptlrpc_run_failed_import_upcall(imp);
-        }
+        CDEBUG(D_HA, "%s: starting recovery\n", obd2cli_tgt(imp->imp_obd));
+        ptlrpc_connect_import(imp);
 
         EXIT;
 }
 
+/**
+ * Identify what request from replay list needs to be replayed next
+ * (based on what we have already replayed) and send it to server.
+ */
 int ptlrpc_replay_next(struct obd_import *imp, int *inflight)
 {
         int rc = 0;
-        struct list_head *tmp, *pos;
+       struct list_head *tmp, *pos;
         struct ptlrpc_request *req = NULL;
-        unsigned long flags;
         __u64 last_transno;
         ENTRY;
 
@@ -161,70 +78,103 @@ int ptlrpc_replay_next(struct obd_import *imp, int *inflight)
         /* It might have committed some after we last spoke, so make sure we
          * get rid of them now.
          */
-        spin_lock_irqsave(&imp->imp_lock, flags);
-        ptlrpc_free_committed(imp);
-        last_transno = imp->imp_last_replay_transno;
-        spin_unlock_irqrestore(&imp->imp_lock, flags);
-
-        CDEBUG(D_HA, "import %p from %s committed "LPU64" last "LPU64"\n",
-               imp, imp->imp_target_uuid.uuid, imp->imp_peer_committed_transno,
-               last_transno);
-
-        /* Do I need to hold a lock across this iteration?  We shouldn't be
-         * racing with any additions to the list, because we're in recovery
-         * and are therefore not processing additional requests to add.  Calls
-         * to ptlrpc_free_committed might commit requests, but nothing "newer"
-         * than the one we're replaying (it can't be committed until it's
-         * replayed, and we're doing that here).  l_f_e_safe protects against
-         * problems with the current request being committed, in the unlikely
-         * event of that race.  So, in conclusion, I think that it's safe to
-         * perform this list-walk without the imp_lock held.
-         *
-         * But, the {mdc,osc}_replay_open callbacks both iterate
-         * request lists, and have comments saying they assume the
-         * imp_lock is being held by ptlrpc_replay, but it's not. it's
-         * just a little race...
-         */
-        list_for_each_safe(tmp, pos, &imp->imp_replay_list) {
-                req = list_entry(tmp, struct ptlrpc_request, rq_replay_list);
-
-                /* If need to resend, stop on the matching one first. It's 
-                   possible though it's already been committed, so in that case 
-                   we'll just continue with replay */
-                if (imp->imp_resend_replay && 
-                    req->rq_transno == last_transno) {
-                        lustre_msg_add_flags(req->rq_reqmsg, MSG_RESENT);
-                        break;
-                }
-
-                if (req->rq_transno > last_transno) {
-                        imp->imp_last_replay_transno = req->rq_transno;
-                        break;
-                }
-
-                req = NULL;
-        }
-
-        imp->imp_resend_replay = 0;
-
-        if (req != NULL) {
-                rc = ptlrpc_replay_req(req);
-                if (rc) {
-                        CERROR("recovery replay error %d for req "
-                               LPD64"\n", rc, req->rq_xid);
-                        RETURN(rc);
-                }
-                imp->imp_reqs_replayed++;
-                *inflight = 1;
-        }
-        RETURN(rc);
+       spin_lock(&imp->imp_lock);
+       imp->imp_last_transno_checked = 0;
+       ptlrpc_free_committed(imp);
+       last_transno = imp->imp_last_replay_transno;
+
+       CDEBUG(D_HA, "import %p from %s committed %llu last %llu\n",
+              imp, obd2cli_tgt(imp->imp_obd),
+              imp->imp_peer_committed_transno, last_transno);
+
+       /* Replay all the committed open requests on committed_list first */
+       if (!list_empty(&imp->imp_committed_list)) {
+               tmp = imp->imp_committed_list.prev;
+               req = list_entry(tmp, struct ptlrpc_request,
+                                    rq_replay_list);
+
+               /* The last request on committed_list hasn't been replayed */
+               if (req->rq_transno > last_transno) {
+                       if (!imp->imp_resend_replay ||
+                           imp->imp_replay_cursor == &imp->imp_committed_list)
+                               imp->imp_replay_cursor =
+                                       imp->imp_replay_cursor->next;
+
+                       while (imp->imp_replay_cursor !=
+                              &imp->imp_committed_list) {
+                               req = list_entry(imp->imp_replay_cursor,
+                                                    struct ptlrpc_request,
+                                                    rq_replay_list);
+                               if (req->rq_transno > last_transno)
+                                       break;
+
+                               req = NULL;
+                               LASSERT(!list_empty(imp->imp_replay_cursor));
+                               imp->imp_replay_cursor =
+                                       imp->imp_replay_cursor->next;
+                       }
+               } else {
+                       /* All requests on committed_list have been replayed */
+                       imp->imp_replay_cursor = &imp->imp_committed_list;
+                       req = NULL;
+               }
+       }
+
+       /* All the requests in committed list have been replayed, let's replay
+        * the imp_replay_list */
+       if (req == NULL) {
+               list_for_each_safe(tmp, pos, &imp->imp_replay_list) {
+                       req = list_entry(tmp, struct ptlrpc_request,
+                                            rq_replay_list);
+
+                       if (req->rq_transno > last_transno)
+                               break;
+                       req = NULL;
+               }
+       }
+
+       /* If need to resend the last sent transno (because a reconnect
+        * has occurred), then stop on the matching req and send it again.
+        * If, however, the last sent transno has been committed then we
+        * continue replay from the next request. */
+       if (req != NULL && imp->imp_resend_replay)
+               lustre_msg_add_flags(req->rq_reqmsg, MSG_RESENT);
+
+       /* ptlrpc_prepare_replay() may fail to add the reqeust into unreplied
+        * list if the request hasn't been added to replay list then. Another
+        * exception is that resend replay could have been removed from the
+        * unreplied list. */
+       if (req != NULL && list_empty(&req->rq_unreplied_list)) {
+               DEBUG_REQ(D_HA, req, "resend_replay: %d, last_transno: %llu\n",
+                         imp->imp_resend_replay, last_transno);
+               ptlrpc_add_unreplied(req);
+               imp->imp_known_replied_xid = ptlrpc_known_replied_xid(imp);
+       }
+
+       imp->imp_resend_replay = 0;
+       spin_unlock(&imp->imp_lock);
+
+       if (req != NULL) {
+               LASSERT(!list_empty(&req->rq_unreplied_list));
+
+               rc = ptlrpc_replay_req(req);
+               if (rc) {
+                       CERROR("recovery replay error %d for req "
+                              "%llu\n", rc, req->rq_xid);
+                       RETURN(rc);
+               }
+               *inflight = 1;
+       }
+       RETURN(rc);
 }
 
+/**
+ * Schedule resending of request on sending_list. This is done after
+ * we completed replaying of requests and locks.
+ */
 int ptlrpc_resend(struct obd_import *imp)
 {
-        struct list_head *tmp, *pos;
-        struct ptlrpc_request *req;
-        unsigned long flags;
+        struct ptlrpc_request *req, *next;
 
         ENTRY;
 
@@ -234,207 +184,203 @@ int ptlrpc_resend(struct obd_import *imp)
          */
         /* Well... what if lctl recover is called twice at the same time?
          */
-        spin_lock_irqsave(&imp->imp_lock, flags);
-        if (imp->imp_state != LUSTRE_IMP_RECOVER) {
-                spin_unlock_irqrestore(&imp->imp_lock, flags);
+       spin_lock(&imp->imp_lock);
+       if (imp->imp_state != LUSTRE_IMP_RECOVER) {
+               spin_unlock(&imp->imp_lock);
                 RETURN(-1);
         }
-        spin_unlock_irqrestore(&imp->imp_lock, flags);
-
-        list_for_each_safe(tmp, pos, &imp->imp_sending_list) {
-                req = list_entry(tmp, struct ptlrpc_request, rq_list);
-                ptlrpc_resend_req(req);
-        }
 
-        RETURN(0);
+       list_for_each_entry_safe(req, next, &imp->imp_sending_list, rq_list) {
+               LASSERTF((long)req > PAGE_SIZE && req != LP_POISON,
+                        "req %p bad\n", req);
+               LASSERTF(req->rq_type != LI_POISON, "req %p freed\n", req);
+
+               /* If the request is allowed to be sent during replay and it
+                * is not timeout yet, then it does not need to be resent. */
+               if (!ptlrpc_no_resend(req) &&
+                   (req->rq_timedout || !req->rq_allow_replay))
+                       ptlrpc_resend_req(req);
+       }
+       spin_unlock(&imp->imp_lock);
+
+       OBD_FAIL_TIMEOUT(OBD_FAIL_LDLM_ENQUEUE_OLD_EXPORT, 2);
+       RETURN(0);
 }
 
+/**
+ * Go through all requests in delayed list and wake their threads
+ * for resending
+ */
 void ptlrpc_wake_delayed(struct obd_import *imp)
 {
-        unsigned long flags;
-        struct list_head *tmp, *pos;
-        struct ptlrpc_request *req;
+       struct list_head *tmp, *pos;
+       struct ptlrpc_request *req;
 
-        spin_lock_irqsave(&imp->imp_lock, flags);
-        list_for_each_safe(tmp, pos, &imp->imp_delayed_list) {
-                req = list_entry(tmp, struct ptlrpc_request, rq_list);
+       spin_lock(&imp->imp_lock);
+       list_for_each_safe(tmp, pos, &imp->imp_delayed_list) {
+               req = list_entry(tmp, struct ptlrpc_request, rq_list);
 
-                DEBUG_REQ(D_HA, req, "waking (set %p):", req->rq_set);
-                ptlrpc_wake_client_req(req);
-        }
-        spin_unlock_irqrestore(&imp->imp_lock, flags);
+               DEBUG_REQ(D_HA, req, "waking (set %p):", req->rq_set);
+               ptlrpc_client_wake_req(req);
+       }
+       spin_unlock(&imp->imp_lock);
 }
 
 void ptlrpc_request_handle_notconn(struct ptlrpc_request *failed_req)
 {
-        int rc;
-        struct obd_import *imp= failed_req->rq_import;
-        unsigned long flags;
+        struct obd_import *imp = failed_req->rq_import;
         ENTRY;
 
         CDEBUG(D_HA, "import %s of %s@%s abruptly disconnected: reconnecting\n",
-               imp->imp_obd->obd_name,
-               imp->imp_target_uuid.uuid,
+               imp->imp_obd->obd_name, obd2cli_tgt(imp->imp_obd),
                imp->imp_connection->c_remote_uuid.uuid);
-        
-        if (ptlrpc_set_import_discon(imp)) {
+
+        if (ptlrpc_set_import_discon(imp,
+                              lustre_msg_get_conn_cnt(failed_req->rq_reqmsg))) {
                 if (!imp->imp_replayable) {
                         CDEBUG(D_HA, "import %s@%s for %s not replayable, "
                                "auto-deactivating\n",
-                               imp->imp_target_uuid.uuid,
+                               obd2cli_tgt(imp->imp_obd),
                                imp->imp_connection->c_remote_uuid.uuid,
                                imp->imp_obd->obd_name);
                         ptlrpc_deactivate_import(imp);
                 }
-
                 /* to control recovery via lctl {disable|enable}_recovery */
                 if (imp->imp_deactive == 0)
-                        rc = ptlrpc_connect_import(imp, NULL);
+                        ptlrpc_connect_import(imp);
         }
 
         /* Wait for recovery to complete and resend. If evicted, then
            this request will be errored out later.*/
-        spin_lock_irqsave(&failed_req->rq_lock, flags);
-        failed_req->rq_resend = 1;
-        spin_unlock_irqrestore(&failed_req->rq_lock, flags);
+       spin_lock(&failed_req->rq_lock);
+       if (!failed_req->rq_no_resend)
+               failed_req->rq_resend = 1;
+       spin_unlock(&failed_req->rq_lock);
 
-        EXIT;
+       EXIT;
 }
 
-/*
+/**
+ * Administratively active/deactive a client. 
  * This should only be called by the ioctl interface, currently
- * with the lctl deactivate and activate commands.
+ *  - the lctl deactivate and activate commands
+ *  - echo 0/1 >> /proc/osc/XXX/active
+ *  - client umount -f (ll_umount_begin)
  */
 int ptlrpc_set_import_active(struct obd_import *imp, int active)
 {
         struct obd_device *obd = imp->imp_obd;
         int rc = 0;
 
+        ENTRY;
         LASSERT(obd);
 
         /* When deactivating, mark import invalid, and abort in-flight
          * requests. */
         if (!active) {
-                ptlrpc_invalidate_import(imp, 0);
-                imp->imp_deactive = 1;
+                LCONSOLE_WARN("setting import %s INACTIVE by administrator "
+                              "request\n", obd2cli_tgt(imp->imp_obd));
+
+                /* set before invalidate to avoid messages about imp_inval
+                 * set without imp_deactive in ptlrpc_import_delay_req */
+               spin_lock(&imp->imp_lock);
+               imp->imp_deactive = 1;
+               spin_unlock(&imp->imp_lock);
+
+                obd_import_event(imp->imp_obd, imp, IMP_EVENT_DEACTIVATE);
+
+                ptlrpc_invalidate_import(imp);
         }
 
         /* When activating, mark import valid, and attempt recovery */
         if (active) {
-                imp->imp_deactive = 0;
                 CDEBUG(D_HA, "setting import %s VALID\n",
-                       imp->imp_target_uuid.uuid);
-                rc = ptlrpc_recover_import(imp, NULL);
-        }
-
-        RETURN(rc);
-}
+                       obd2cli_tgt(imp->imp_obd));
 
-int ptlrpc_recover_import(struct obd_import *imp, char *new_uuid)
-{
-        int rc;
-        ENTRY;
-
-        /* force import to be disconnected. */
-        ptlrpc_set_import_discon(imp);
+               spin_lock(&imp->imp_lock);
+               imp->imp_deactive = 0;
+               spin_unlock(&imp->imp_lock);
+                obd_import_event(imp->imp_obd, imp, IMP_EVENT_ACTIVATE);
 
-        rc = ptlrpc_recover_import_no_retry(imp, new_uuid);
+                rc = ptlrpc_recover_import(imp, NULL, 0);
+        }
 
         RETURN(rc);
 }
+EXPORT_SYMBOL(ptlrpc_set_import_active);
 
-int ptlrpc_import_in_recovery(struct obd_import *imp)
+/* Attempt to reconnect an import */
+int ptlrpc_recover_import(struct obd_import *imp, char *new_uuid, int async)
 {
-        unsigned long flags;
-        int in_recovery = 1;
-        spin_lock_irqsave(&imp->imp_lock, flags);
-        if (imp->imp_state == LUSTRE_IMP_FULL ||
-            imp->imp_state == LUSTRE_IMP_CLOSED ||
-            imp->imp_state == LUSTRE_IMP_DISCON)
-                in_recovery = 0;
-        spin_unlock_irqrestore(&imp->imp_lock, flags);
-        return in_recovery;
-}
+       int rc = 0;
+       ENTRY;
+
+       spin_lock(&imp->imp_lock);
+       if (imp->imp_state == LUSTRE_IMP_NEW || imp->imp_deactive ||
+           atomic_read(&imp->imp_inval_count))
+               rc = -EINVAL;
+       spin_unlock(&imp->imp_lock);
+        if (rc)
+                GOTO(out, rc);
 
-int ptlrpc_import_control_recovery(struct obd_import *imp, int disable)
-{
-        unsigned long flags;
-
-        /* with imp_deactivate == 1 pinger won't initiate re-connect */
-        spin_lock_irqsave(&imp->imp_lock, flags);
-        if (disable)
-                imp->imp_deactive = 1;
-        else
-                imp->imp_deactive = 0;
-        if (imp->imp_state == LUSTRE_IMP_DISCON) {
-                imp->imp_force_verify = 1;
-                ptlrpc_pinger_wake_up();
-        }
-        spin_unlock_irqrestore(&imp->imp_lock, flags);
-        RETURN(0);
-}
+        /* force import to be disconnected. */
+        ptlrpc_set_import_discon(imp, 0);
 
-static int ptlrpc_recover_import_no_retry(struct obd_import *imp,
-                                          char *new_uuid)
-{
-        int rc;
-        unsigned long flags;
-        int in_recovery = 0;
-        struct l_wait_info lwi;
-        ENTRY;
+        if (new_uuid) {
+                struct obd_uuid uuid;
 
-        spin_lock_irqsave(&imp->imp_lock, flags);
-        if (imp->imp_state != LUSTRE_IMP_DISCON) {
-                in_recovery = 1;
+                /* intruct import to use new uuid */
+                obd_str2uuid(&uuid, new_uuid);
+                rc = import_set_conn_priority(imp, &uuid);
+                if (rc)
+                        GOTO(out, rc);
         }
-        spin_unlock_irqrestore(&imp->imp_lock, flags);
 
-        if (in_recovery == 1)
-                RETURN(-EALREADY);
+        /* Check if reconnect is already in progress */
+       spin_lock(&imp->imp_lock);
+       if (imp->imp_state != LUSTRE_IMP_DISCON) {
+               imp->imp_force_verify = 1;
+               rc = -EALREADY;
+       }
+       spin_unlock(&imp->imp_lock);
+        if (rc)
+                GOTO(out, rc);
 
-        rc = ptlrpc_connect_import(imp, new_uuid);
+        rc = ptlrpc_connect_import(imp);
         if (rc)
-                RETURN(rc);
+                GOTO(out, rc);
 
-        CDEBUG(D_HA, "%s: recovery started, waiting\n",
-               imp->imp_target_uuid.uuid);
+        if (!async) {
+                struct l_wait_info lwi;
+                int secs = cfs_time_seconds(obd_timeout);
 
-        lwi = LWI_TIMEOUT(MAX(obd_timeout * HZ, 1), NULL, NULL);
-        rc = l_wait_event(imp->imp_recovery_waitq,
-                          !ptlrpc_import_in_recovery(imp), &lwi);
-        CDEBUG(D_HA, "%s: recovery finished\n",
-               imp->imp_target_uuid.uuid);
+                CDEBUG(D_HA, "%s: recovery started, waiting %u seconds\n",
+                       obd2cli_tgt(imp->imp_obd), secs);
 
-        RETURN(rc);
+                lwi = LWI_TIMEOUT(secs, NULL, NULL);
+                rc = l_wait_event(imp->imp_recovery_waitq,
+                                  !ptlrpc_import_in_recovery(imp), &lwi);
+                CDEBUG(D_HA, "%s: recovery finished\n",
+                       obd2cli_tgt(imp->imp_obd));
+        }
+        EXIT;
+
+out:
+        return rc;
 }
+EXPORT_SYMBOL(ptlrpc_recover_import);
 
-void ptlrpc_fail_export(struct obd_export *exp)
+int ptlrpc_import_in_recovery(struct obd_import *imp)
 {
-        int rc, already_failed;
-        unsigned long flags;
-
-        spin_lock_irqsave(&exp->exp_lock, flags);
-        already_failed = exp->exp_failed;
-        exp->exp_failed = 1;
-        spin_unlock_irqrestore(&exp->exp_lock, flags);
-
-        if (already_failed) {
-                CDEBUG(D_HA, "disconnecting dead export %p/%s; skipping\n",
-                       exp, exp->exp_client_uuid.uuid);
-                return;
-        }
+       int in_recovery = 1;
 
-        CDEBUG(D_HA, "disconnecting export %p/%s\n",
-               exp, exp->exp_client_uuid.uuid);
+       spin_lock(&imp->imp_lock);
+       if (imp->imp_state == LUSTRE_IMP_FULL ||
+           imp->imp_state == LUSTRE_IMP_CLOSED ||
+           imp->imp_state == LUSTRE_IMP_DISCON ||
+           imp->imp_obd->obd_no_recov)
+               in_recovery = 0;
+       spin_unlock(&imp->imp_lock);
 
-        if (obd_dump_on_timeout)
-                portals_debug_dumplog();
-
-        /* Most callers into obd_disconnect are removing their own reference
-         * (request, for example) in addition to the one from the hash table.
-         * We don't have such a reference here, so make one. */
-        class_export_get(exp);
-        rc = obd_disconnect(exp, 0);
-        if (rc)
-                CERROR("disconnecting export %p failed: %d\n", exp, rc);
+       return in_recovery;
 }