Whamcloud - gitweb
LU-1095 debug: conditionally print recovery status
[fs/lustre-release.git] / lustre / ldlm / ldlm_lib.c
index 4487d5a..54780a7 100644 (file)
@@ -34,9 +34,6 @@
  * Lustre is a trademark of Sun Microsystems, Inc.
  */
 
-#ifndef EXPORT_SYMTAB
-# define EXPORT_SYMTAB
-#endif
 #define DEBUG_SUBSYSTEM S_LDLM
 
 #ifdef __KERNEL__
@@ -45,7 +42,7 @@
 # include <liblustre.h>
 #endif
 #include <obd.h>
-#include <lustre_mds.h>
+#include <obd_class.h>
 #include <lustre_dlm.h>
 #include <lustre_net.h>
 #include <lustre_sec.h>
@@ -226,6 +223,29 @@ void client_destroy_import(struct obd_import *imp)
 }
 EXPORT_SYMBOL(client_destroy_import);
 
+/**
+ * check whether the osc is on MDT or not
+ * In the config log,
+ * osc on MDT
+ *     setup 0:{fsname}-OSTxxxx-osc[-MDTxxxx] 1:lustre-OST0000_UUID 2:NID
+ * osc on client
+ *     setup 0:{fsname}-OSTxxxx-osc 1:lustre-OST0000_UUID 2:NID
+ *
+ **/
+static int osc_on_mdt(char *obdname)
+{
+       char *ptr;
+
+       ptr = strrchr(obdname, '-');
+       if (ptr == NULL)
+               return 0;
+
+       if (strncmp(ptr + 1, "MDT", 3) == 0)
+               return 1;
+
+       return 0;
+}
+
 /* configure an RPC client OBD device
  *
  * lcfg parameters:
@@ -246,7 +266,9 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg)
 
         /* In a more perfect world, we would hang a ptlrpc_client off of
          * obd_type and just use the values from there. */
-        if (!strcmp(name, LUSTRE_OSC_NAME)) {
+       if (!strcmp(name, LUSTRE_OSC_NAME) ||
+           (!strcmp(name, LUSTRE_OSP_NAME) &&
+            !is_osp_on_ost(lustre_cfg_buf(lcfg, 0)))) {
                 rq_portal = OST_REQUEST_PORTAL;
                 rp_portal = OSC_REPLY_PORTAL;
                 connect_op = OST_CONNECT;
@@ -254,7 +276,9 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg)
                 cli->cl_sp_to = LUSTRE_SP_OST;
                 ns_type = LDLM_NS_TYPE_OSC;
 
-        } else if (!strcmp(name, LUSTRE_MDC_NAME)) {
+       } else if (!strcmp(name, LUSTRE_MDC_NAME) ||
+                  (!strcmp(name, LUSTRE_OSP_NAME) &&
+                   is_osp_on_ost(lustre_cfg_buf(lcfg, 0)))) {
                 rq_portal = MDS_REQUEST_PORTAL;
                 rp_portal = MDC_REPLY_PORTAL;
                 connect_op = MDS_CONNECT;
@@ -271,6 +295,14 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg)
                 cli->cl_flvr_mgc.sf_rpc = SPTLRPC_FLVR_INVALID;
                 ns_type = LDLM_NS_TYPE_MGC;
 
+       } else if (!strcmp(name, LUSTRE_OSP_NAME)) {
+               rq_portal = OST_REQUEST_PORTAL;
+               rp_portal = OSC_REPLY_PORTAL;
+               connect_op = OST_CONNECT;
+               cli->cl_sp_me = LUSTRE_SP_CLI;
+               cli->cl_sp_to = LUSTRE_SP_OST;
+               ns_type = LDLM_NS_TYPE_OSC;
+
         } else {
                 CERROR("unknown client OBD type \"%s\", can't setup\n",
                        name);
@@ -316,6 +348,8 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg)
         CFS_INIT_LIST_HEAD(&cli->cl_loi_write_list);
         CFS_INIT_LIST_HEAD(&cli->cl_loi_read_list);
         client_obd_list_lock_init(&cli->cl_loi_list_lock);
+       cfs_atomic_set(&cli->cl_pending_w_pages, 0);
+       cfs_atomic_set(&cli->cl_pending_r_pages, 0);
         cli->cl_r_in_flight = 0;
         cli->cl_w_in_flight = 0;
 
@@ -353,9 +387,11 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg)
         } else if (cfs_num_physpages >> (20 - CFS_PAGE_SHIFT) <= 512 /* MB */) {
                 cli->cl_max_rpcs_in_flight = 4;
         } else {
-                cli->cl_max_rpcs_in_flight = OSC_MAX_RIF_DEFAULT;
+               if (osc_on_mdt(obddev->obd_name))
+                       cli->cl_max_rpcs_in_flight = MDS_OSC_MAX_RIF_DEFAULT;
+               else
+                       cli->cl_max_rpcs_in_flight = OSC_MAX_RIF_DEFAULT;
         }
-
         rc = ldlm_get_ref();
         if (rc) {
                 CERROR("ldlm_get_ref failed: %d\n", rc);
@@ -596,18 +632,20 @@ int server_disconnect_export(struct obd_export *exp)
                 struct ptlrpc_reply_state *rs =
                         cfs_list_entry(exp->exp_outstanding_replies.next,
                                        struct ptlrpc_reply_state, rs_exp_list);
-                struct ptlrpc_service *svc = rs->rs_service;
+               struct ptlrpc_service_part *svcpt = rs->rs_svcpt;
 
-                cfs_spin_lock(&svc->srv_rs_lock);
-                cfs_list_del_init(&rs->rs_exp_list);
-                cfs_spin_lock(&rs->rs_lock);
-                ptlrpc_schedule_difficult_reply(rs);
-                cfs_spin_unlock(&rs->rs_lock);
-                cfs_spin_unlock(&svc->srv_rs_lock);
-        }
-        cfs_spin_unlock(&exp->exp_lock);
+               cfs_spin_lock(&svcpt->scp_rep_lock);
 
-        RETURN(rc);
+               cfs_list_del_init(&rs->rs_exp_list);
+               cfs_spin_lock(&rs->rs_lock);
+               ptlrpc_schedule_difficult_reply(rs);
+               cfs_spin_unlock(&rs->rs_lock);
+
+               cfs_spin_unlock(&svcpt->scp_rep_lock);
+       }
+       cfs_spin_unlock(&exp->exp_lock);
+
+       RETURN(rc);
 }
 
 /* --------------------------------------------------------------------------
@@ -655,14 +693,13 @@ static int target_handle_reconnect(struct lustre_handle *conn,
                          * doing a valid reconnect from the same client. */
                         RETURN(EALREADY);
                 } else {
-                        LCONSOLE_WARN("%s: The server has already connected "
-                                      "client %s (at %s) with handle " LPX64
-                                      ", rejecting a client with the same "
-                                      "uuid trying to reconnect with "
-                                      "handle " LPX64, target->obd_name,
-                                      obd_uuid2str(&exp->exp_client_uuid),
-                                      obd_export_nid2str(exp),
-                                      hdl->cookie, conn->cookie);
+                       LCONSOLE_WARN("%s: already connected client %s (at %s) "
+                                     "with handle "LPX64". Rejecting client "
+                                     "with the same UUID trying to reconnect "
+                                     "with handle "LPX64"\n", target->obd_name,
+                                     obd_uuid2str(&exp->exp_client_uuid),
+                                     obd_export_nid2str(exp),
+                                     hdl->cookie, conn->cookie);
                         memset(conn, 0, sizeof *conn);
                         /* target_handle_connect() treats EALREADY and
                          * -EALREADY differently.  -EALREADY is an error
@@ -706,9 +743,10 @@ check_and_start_recovery_timer(struct obd_device *obd,
 
 int target_handle_connect(struct ptlrpc_request *req)
 {
-        struct obd_device *target, *targref = NULL;
+       struct obd_device *target = NULL, *targref = NULL;
         struct obd_export *export = NULL;
         struct obd_import *revimp;
+       struct obd_import *tmp_imp = NULL;
         struct lustre_handle conn;
         struct lustre_handle *tmp;
         struct obd_uuid tgtuuid;
@@ -718,11 +756,11 @@ int target_handle_connect(struct ptlrpc_request *req)
         int rc = 0;
         char *target_start;
         int target_len;
-        int mds_conn = 0;
+       bool     mds_conn = false, lw_client = false;
         struct obd_connect_data *data, *tmpdata;
         int size, tmpsize;
         lnet_nid_t *client_nid = NULL;
-        ENTRY;
+       ENTRY;
 
         OBD_RACE(OBD_FAIL_TGT_CONN_RACE);
 
@@ -737,17 +775,29 @@ int target_handle_connect(struct ptlrpc_request *req)
         if (!target)
                 target = class_name2obd(str);
 
-        if (!target || target->obd_stopping || !target->obd_set_up) {
-                deuuidify(str, NULL, &target_start, &target_len);
-                LCONSOLE_ERROR_MSG(0x137, "%.*s: Not available for connect "
-                                   "from %s (%s)\n", target_len, target_start,
-                                   libcfs_nid2str(req->rq_peer.nid), !target ?
-                                   "no target" : (target->obd_stopping ?
-                                   "stopping" : "not set up"));
-                GOTO(out, rc = -ENODEV);
-        }
+       if (!target) {
+               deuuidify(str, NULL, &target_start, &target_len);
+               LCONSOLE_ERROR_MSG(0x137, "UUID '%s' is not available for "
+                                  "connect (no target)\n", str);
+               GOTO(out, rc = -ENODEV);
+       }
+
+       cfs_spin_lock(&target->obd_dev_lock);
+       if (target->obd_stopping || !target->obd_set_up) {
+               cfs_spin_unlock(&target->obd_dev_lock);
+
+               deuuidify(str, NULL, &target_start, &target_len);
+               LCONSOLE_ERROR_MSG(0x137, "%.*s: Not available for connect "
+                                  "from %s (%s)\n", target_len, target_start,
+                                  libcfs_nid2str(req->rq_peer.nid), 
+                                  (target->obd_stopping ?
+                                  "stopping" : "not set up"));
+               GOTO(out, rc = -ENODEV);
+       }
 
         if (target->obd_no_conn) {
+               cfs_spin_unlock(&target->obd_dev_lock);
+
                 LCONSOLE_WARN("%s: Temporarily refusing client connection "
                               "from %s\n", target->obd_name,
                               libcfs_nid2str(req->rq_peer.nid));
@@ -759,6 +809,8 @@ int target_handle_connect(struct ptlrpc_request *req)
            Really, class_uuid2obd should take the ref. */
         targref = class_incref(target, __FUNCTION__, cfs_current());
 
+       target->obd_conn_inprogress++;
+       cfs_spin_unlock(&target->obd_dev_lock);
 
         str = req_capsule_client_get(&req->rq_pill, &RMF_CLUUID);
         if (str == NULL) {
@@ -830,7 +882,10 @@ int target_handle_connect(struct ptlrpc_request *req)
 
         if ((lustre_msg_get_op_flags(req->rq_reqmsg) & MSG_CONNECT_INITIAL) &&
             (data->ocd_connect_flags & OBD_CONNECT_MDS))
-                mds_conn = 1;
+               mds_conn = true;
+
+       if ((data->ocd_connect_flags & OBD_CONNECT_LIGHTWEIGHT) != 0)
+               lw_client = true;
 
         /* lctl gets a backstage, all-access pass. */
         if (obd_uuid_equals(&cluuid, &target->obd_uuid))
@@ -841,7 +896,11 @@ int target_handle_connect(struct ptlrpc_request *req)
                 goto no_export;
 
         /* we've found an export in the hash */
+
+       cfs_spin_lock(&export->exp_lock);
+
         if (export->exp_connecting) { /* bug 9635, et. al. */
+               cfs_spin_unlock(&export->exp_lock);
                 LCONSOLE_WARN("%s: Export %p already connecting from %s\n",
                               export->exp_obd->obd_name, export,
                               libcfs_nid2str(req->rq_peer.nid));
@@ -849,6 +908,7 @@ int target_handle_connect(struct ptlrpc_request *req)
                 export = NULL;
                 rc = -EALREADY;
         } else if (mds_conn && export->exp_connection) {
+               cfs_spin_unlock(&export->exp_lock);
                 if (req->rq_peer.nid != export->exp_connection->c_peer.nid)
                         /* mds reconnected after failover */
                         LCONSOLE_WARN("%s: Received MDS connection from "
@@ -868,6 +928,7 @@ int target_handle_connect(struct ptlrpc_request *req)
                    req->rq_peer.nid != export->exp_connection->c_peer.nid &&
                    (lustre_msg_get_op_flags(req->rq_reqmsg) &
                     MSG_CONNECT_INITIAL)) {
+               cfs_spin_unlock(&export->exp_lock);
                 /* in mds failover we have static uuid but nid can be
                  * changed*/
                 LCONSOLE_WARN("%s: Client %s seen on new nid %s when "
@@ -880,13 +941,12 @@ int target_handle_connect(struct ptlrpc_request *req)
                 class_export_put(export);
                 export = NULL;
         } else {
-                cfs_spin_lock(&export->exp_lock);
-                export->exp_connecting = 1;
-                cfs_spin_unlock(&export->exp_lock);
-                LASSERT(export->exp_obd == target);
+               export->exp_connecting = 1;
+               cfs_spin_unlock(&export->exp_lock);
+               LASSERT(export->exp_obd == target);
 
-                rc = target_handle_reconnect(&conn, export, &cluuid);
-        }
+               rc = target_handle_reconnect(&conn, export, &cluuid);
+       }
 
         /* If we found an export, we already unlocked. */
         if (!export) {
@@ -938,8 +998,8 @@ no_export:
               export ? (long)export->exp_last_request_time : 0);
 
         /* If this is the first time a client connects, reset the recovery
-         * timer */
-        if (rc == 0 && target->obd_recovering)
+        * timer. Discard lightweight connections which might be local */
+       if (!lw_client && rc == 0 && target->obd_recovering)
                 check_and_start_recovery_timer(target, req, export == NULL);
 
         /* We want to handle EALREADY but *not* -EALREADY from
@@ -957,44 +1017,57 @@ no_export:
         client_nid = &req->rq_peer.nid;
 
         if (export == NULL) {
-                if (target->obd_recovering) {
+               /* allow lightweight connections during recovery */
+               if (target->obd_recovering && !lw_client) {
                         cfs_time_t t;
-
-                        t = cfs_timer_deadline(&target->obd_recovery_timer);
-                        t = cfs_time_sub(t, cfs_time_current());
-                        t = cfs_duration_sec(t);
-                        LCONSOLE_WARN("%s: Denying connection for new client "
-                                      "%s (at %s), waiting for %d clients in "
-                                      "recovery for %d:%.02d\n",
-                                      target->obd_name,
-                                      libcfs_nid2str(req->rq_peer.nid),
-                                      cluuid.uuid,
-                                      cfs_atomic_read(&target-> \
-                                                      obd_lock_replay_clients),
-                                      (int)t / 60, (int)t % 60);
+                       int        c; /* connected */
+                       int        i; /* in progress */
+                       int        k; /* known */
+
+                       c = cfs_atomic_read(&target->obd_connected_clients);
+                       i = cfs_atomic_read(&target->obd_lock_replay_clients);
+                       k = target->obd_max_recoverable_clients;
+                       t = cfs_timer_deadline(&target->obd_recovery_timer);
+                       t = cfs_time_sub(t, cfs_time_current());
+                       t = cfs_duration_sec(t);
+                       LCONSOLE_WARN("%s: Denying connection for new client "
+                                     "%s (at %s), waiting for all %d known "
+                                     "clients (%d recovered, %d in progress, "
+                                     "and %d unseen) to recover in %d:%.02d\n",
+                                     target->obd_name, cluuid.uuid,
+                                     libcfs_nid2str(req->rq_peer.nid), k,
+                                     c - i, i, k - c, (int)t / 60,
+                                     (int)t % 60);
                         rc = -EBUSY;
                 } else {
 dont_check_exports:
                         rc = obd_connect(req->rq_svc_thread->t_env,
                                          &export, target, &cluuid, data,
                                          client_nid);
-                        if (rc == 0) {
+                        if (rc == 0)
                                 conn.cookie = export->exp_handle.h_cookie;
-                                /* LU-1092 reconnect put export refcount in the
-                                 * end, connect needs take one here too. */
-                                class_export_get(export);
-                        }
                 }
         } else {
                 rc = obd_reconnect(req->rq_svc_thread->t_env,
                                    export, target, &cluuid, data, client_nid);
-                if (rc == 0)
-                        /* prevous done via class_conn2export */
-                        class_export_get(export);
         }
         if (rc)
                 GOTO(out, rc);
 
+#if LUSTRE_VERSION_CODE < OBD_OCD_VERSION(2, 6, 50, 0)
+       /* 2.2.0 clients always swab nidtbl entries due to a bug, so server
+        * will do the swabbing for if the client is using the same endianness.
+        *
+        * This fixup is version-limited, because we don't want to carry the
+        * OBD_CONNECT_MNE_SWAB flag around forever, just so long as we need
+        * interop with unpatched 2.2 clients.  For newer clients, servers
+        * will never do MNE swabbing, let the client handle that.  LU-1644 */
+       export->exp_need_mne_swab = !ptlrpc_req_need_swab(req) &&
+                       !(data->ocd_connect_flags & OBD_CONNECT_MNE_SWAB);
+#else
+#warning "LU-1644: Remove old OBD_CONNECT_MNE_SWAB fixup and exp_need_mne_swab"
+#endif
+
         LASSERT(target->u.obt.obt_magic == OBT_MAGIC);
         data->ocd_instance = target->u.obt.obt_instance;
 
@@ -1026,7 +1099,8 @@ dont_check_exports:
         if (req->rq_export != NULL)
                 class_export_put(req->rq_export);
 
-        req->rq_export = export;
+       /* request takes one export refcount */
+       req->rq_export = class_export_get(export);
 
         cfs_spin_lock(&export->exp_lock);
         if (export->exp_conn_cnt >= lustre_msg_get_conn_cnt(req->rq_reqmsg)) {
@@ -1074,14 +1148,18 @@ dont_check_exports:
                              &export->exp_connection->c_peer.nid,
                              &export->exp_nid_hash);
         }
-        /**
-          class_disconnect->class_export_recovery_cleanup() race
-         */
-        if (target->obd_recovering && !export->exp_in_recovery) {
+
+       if (target->obd_recovering && !export->exp_in_recovery && !lw_client) {
                 int has_transno;
                 __u64 transno = data->ocd_transno;
 
                 cfs_spin_lock(&export->exp_lock);
+               /* possible race with class_disconnect_stale_exports,
+                * export may be already in the eviction process */
+               if (export->exp_failed) {
+                       cfs_spin_unlock(&export->exp_lock);
+                       GOTO(out, rc = -ENODEV);
+               }
                 export->exp_in_recovery = 1;
                 export->exp_req_replay_needed = 1;
                 export->exp_lock_replay_needed = 1;
@@ -1111,28 +1189,31 @@ dont_check_exports:
         }
 
         /* Tell the client we're in recovery, when client is involved in it. */
-        if (target->obd_recovering)
+       if (target->obd_recovering && !lw_client)
                 lustre_msg_add_op_flags(req->rq_repmsg, MSG_CONNECT_RECOVERING);
 
         tmp = req_capsule_client_get(&req->rq_pill, &RMF_CONN);
         conn = *tmp;
 
-        if (export->exp_imp_reverse != NULL) {
-                /* destroyed import can be still referenced in ctxt */
-                obd_set_info_async(export, sizeof(KEY_REVIMP_UPD),
-                                   KEY_REVIMP_UPD, 0, NULL, NULL);
-
-                client_destroy_import(export->exp_imp_reverse);
-        }
-
-        /* for the rest part, we return -ENOTCONN in case of errors
-         * in order to let client initialize connection again.
-         */
-        revimp = export->exp_imp_reverse = class_new_import(target);
-        if (!revimp) {
-                CERROR("fail to alloc new reverse import.\n");
-                GOTO(out, rc = -ENOTCONN);
-        }
+       /* for the rest part, we return -ENOTCONN in case of errors
+        * in order to let client initialize connection again.
+        */
+       revimp = class_new_import(target);
+       if (revimp == NULL) {
+               CERROR("fail to alloc new reverse import.\n");
+               GOTO(out, rc = -ENOTCONN);
+       }
+
+       cfs_spin_lock(&export->exp_lock);
+       if (export->exp_imp_reverse != NULL) {
+               /* destroyed import can be still referenced in ctxt */
+               obd_set_info_async(req->rq_svc_thread->t_env, export,
+                                  sizeof(KEY_REVIMP_UPD), KEY_REVIMP_UPD,
+                                  0, NULL, NULL);
+               tmp_imp = export->exp_imp_reverse;
+       }
+       export->exp_imp_reverse = revimp;
+       cfs_spin_unlock(&export->exp_lock);
 
         revimp->imp_connection = ptlrpc_connection_addref(export->exp_connection);
         revimp->imp_client = &export->exp_obd->obd_ldlm_client;
@@ -1156,15 +1237,20 @@ dont_check_exports:
         else
                 revimp->imp_msghdr_flags &= ~MSGHDR_CKSUM_INCOMPAT18;
 
-        rc = sptlrpc_import_sec_adapt(revimp, req->rq_svc_ctx, &req->rq_flvr);
-        if (rc) {
-                CERROR("Failed to get sec for reverse import: %d\n", rc);
-                export->exp_imp_reverse = NULL;
-                class_destroy_import(revimp);
-        }
+       rc = sptlrpc_import_sec_adapt(revimp, req->rq_svc_ctx, &req->rq_flvr);
+       if (rc) {
+               CERROR("Failed to get sec for reverse import: %d\n", rc);
+               cfs_spin_lock(&export->exp_lock);
+               export->exp_imp_reverse = NULL;
+               cfs_spin_unlock(&export->exp_lock);
+               class_destroy_import(revimp);
+       }
+
+       class_import_put(revimp);
 
-        class_import_put(revimp);
 out:
+       if (tmp_imp != NULL)
+               client_destroy_import(tmp_imp);
         if (export) {
                 cfs_spin_lock(&export->exp_lock);
                 export->exp_connecting = 0;
@@ -1172,8 +1258,13 @@ out:
 
                 class_export_put(export);
         }
-        if (targref)
+        if (targref) {
+               cfs_spin_lock(&target->obd_dev_lock);
+               target->obd_conn_inprogress--;
+               cfs_spin_unlock(&target->obd_dev_lock);
+
                 class_decref(targref, __FUNCTION__, cfs_current());
+       }
         if (rc)
                 req->rq_status = rc;
         RETURN(rc);
@@ -1196,15 +1287,22 @@ int target_handle_disconnect(struct ptlrpc_request *req)
 
 void target_destroy_export(struct obd_export *exp)
 {
-        /* exports created from last_rcvd data, and "fake"
-           exports created by lctl don't have an import */
-        if (exp->exp_imp_reverse != NULL)
-                client_destroy_import(exp->exp_imp_reverse);
-
-        LASSERT_ATOMIC_ZERO(&exp->exp_locks_count);
-        LASSERT_ATOMIC_ZERO(&exp->exp_rpc_count);
-        LASSERT_ATOMIC_ZERO(&exp->exp_cb_count);
-        LASSERT_ATOMIC_ZERO(&exp->exp_replay_count);
+       struct obd_import       *imp = NULL;
+       /* exports created from last_rcvd data, and "fake"
+          exports created by lctl don't have an import */
+       cfs_spin_lock(&exp->exp_lock);
+       if (exp->exp_imp_reverse != NULL) {
+               imp = exp->exp_imp_reverse;
+               exp->exp_imp_reverse = NULL;
+       }
+       cfs_spin_unlock(&exp->exp_lock);
+       if (imp != NULL)
+               client_destroy_import(imp);
+
+       LASSERT_ATOMIC_ZERO(&exp->exp_locks_count);
+       LASSERT_ATOMIC_ZERO(&exp->exp_rpc_count);
+       LASSERT_ATOMIC_ZERO(&exp->exp_cb_count);
+       LASSERT_ATOMIC_ZERO(&exp->exp_replay_count);
 }
 
 /*
@@ -1278,17 +1376,20 @@ static void target_exp_dequeue_req_replay(struct ptlrpc_request *req)
 #ifdef __KERNEL__
 static void target_finish_recovery(struct obd_device *obd)
 {
-        time_t elapsed_time = max_t(time_t, 1, cfs_time_current_sec() -
-                                    obd->obd_recovery_start);
         ENTRY;
 
-        LCONSOLE_INFO("%s: Recovery over after %d:%.02d, of %d clients "
-                      "%d recovered and %d %s evicted.\n", obd->obd_name,
-                      (int)elapsed_time / 60, (int)elapsed_time % 60,
-                      obd->obd_max_recoverable_clients,
-                      cfs_atomic_read(&obd->obd_connected_clients),
-                      obd->obd_stale_clients,
-                      obd->obd_stale_clients == 1 ? "was" : "were");
+       /* only log a recovery message when recovery has occurred */
+       if (obd->obd_recovery_start) {
+               time_t elapsed_time = max_t(time_t, 1, cfs_time_current_sec() -
+                                       obd->obd_recovery_start);
+               LCONSOLE_INFO("%s: Recovery over after %d:%.02d, of %d clients "
+                       "%d recovered and %d %s evicted.\n", obd->obd_name,
+                       (int)elapsed_time / 60, (int)elapsed_time % 60,
+                       obd->obd_max_recoverable_clients,
+                       cfs_atomic_read(&obd->obd_connected_clients),
+                       obd->obd_stale_clients,
+                       obd->obd_stale_clients == 1 ? "was" : "were");
+       }
 
         ldlm_reprocess_all_ns(obd->obd_namespace);
         cfs_spin_lock(&obd->obd_recovery_task_lock);
@@ -1518,7 +1619,7 @@ check_and_start_recovery_timer(struct obd_device *obd,
         if (!new_client && service_time)
                 /* Teach server about old server's estimates, as first guess
                  * at how long new requests will take. */
-                at_measured(&req->rq_rqbd->rqbd_service->srv_at_estimate,
+               at_measured(&req->rq_rqbd->rqbd_svcpt->scp_at_estimate,
                             service_time);
 
         target_start_recovery_timer(obd);
@@ -1534,7 +1635,7 @@ check_and_start_recovery_timer(struct obd_device *obd,
 
         LASSERT(obt->obt_magic == OBT_MAGIC);
         lsi = s2lsi(obt->obt_sb);
-        if (!(lsi->lsi_flags | LSI_IR_CAPABLE))
+       if (!(lsi->lsi_flags | LDD_F_IR_CAPABLE))
                 service_time += 2 * (CONNECTION_SWITCH_MAX +
                                      CONNECTION_SWITCH_INC);
         if (service_time > obd->obd_recovery_timeout && !new_client)
@@ -1829,15 +1930,17 @@ static int handle_recovery_req(struct ptlrpc_thread *thread,
                  * this client may come in recovery time
                  */
                 if (!AT_OFF) {
-                        struct ptlrpc_service *svc = req->rq_rqbd->rqbd_service;
-                        /* If the server sent early reply for this request,
-                         * the client will recalculate the timeout according to
-                         * current server estimate service time, so we will
-                         * use the maxium timeout here for waiting the client
-                         * sending the next req */
-                        to = max((int)at_est2timeout(
-                                 at_get(&svc->srv_at_estimate)),
-                                 (int)lustre_msg_get_timeout(req->rq_reqmsg));
+                       struct ptlrpc_service_part *svcpt;
+
+                       svcpt = req->rq_rqbd->rqbd_svcpt;
+                       /* If the server sent early reply for this request,
+                        * the client will recalculate the timeout according to
+                        * current server estimate service time, so we will
+                        * use the maxium timeout here for waiting the client
+                        * sending the next req */
+                       to = max((int)at_est2timeout(
+                                at_get(&svcpt->scp_at_estimate)),
+                                (int)lustre_msg_get_timeout(req->rq_reqmsg));
                         /* Add net_latency (see ptlrpc_replay_req) */
                         to += lustre_msg_get_service_time(req->rq_reqmsg);
                 }
@@ -2240,7 +2343,7 @@ int target_queue_recovery_request(struct ptlrpc_request *req,
 
 int target_handle_ping(struct ptlrpc_request *req)
 {
-        obd_ping(req->rq_export);
+        obd_ping(req->rq_svc_thread->t_env, req->rq_export);
         return req_capsule_server_pack(&req->rq_pill);
 }
 
@@ -2315,10 +2418,10 @@ int target_send_reply_msg(struct ptlrpc_request *req, int rc, int fail_id)
 
 void target_send_reply(struct ptlrpc_request *req, int rc, int fail_id)
 {
+       struct ptlrpc_service_part *svcpt;
         int                        netrc;
         struct ptlrpc_reply_state *rs;
         struct obd_export         *exp;
-        struct ptlrpc_service     *svc;
         ENTRY;
 
         if (req->rq_no_reply) {
@@ -2326,7 +2429,7 @@ void target_send_reply(struct ptlrpc_request *req, int rc, int fail_id)
                 return;
         }
 
-        svc = req->rq_rqbd->rqbd_service;
+       svcpt = req->rq_rqbd->rqbd_svcpt;
         rs = req->rq_reply_state;
         if (rs == NULL || !rs->rs_difficult) {
                 /* no notifiers */
@@ -2338,7 +2441,7 @@ void target_send_reply(struct ptlrpc_request *req, int rc, int fail_id)
         /* must be an export if locks saved */
         LASSERT (req->rq_export != NULL);
         /* req/reply consistent */
-        LASSERT (rs->rs_service == svc);
+       LASSERT(rs->rs_svcpt == svcpt);
 
         /* "fresh" reply */
         LASSERT (!rs->rs_scheduled);
@@ -2375,9 +2478,9 @@ void target_send_reply(struct ptlrpc_request *req, int rc, int fail_id)
 
         netrc = target_send_reply_msg (req, rc, fail_id);
 
-        cfs_spin_lock(&svc->srv_rs_lock);
+       cfs_spin_lock(&svcpt->scp_rep_lock);
 
-        cfs_atomic_inc(&svc->srv_n_difficult_replies);
+       cfs_atomic_inc(&svcpt->scp_nreps_difficult);
 
         if (netrc != 0) {
                 /* error sending: reply is off the net.  Also we need +1
@@ -2397,126 +2500,13 @@ void target_send_reply(struct ptlrpc_request *req, int rc, int fail_id)
                 CDEBUG(D_HA, "Schedule reply immediately\n");
                 ptlrpc_dispatch_difficult_reply(rs);
         } else {
-                cfs_list_add (&rs->rs_list, &svc->srv_active_replies);
-                rs->rs_scheduled = 0;           /* allow notifier to schedule */
-        }
-        cfs_spin_unlock(&rs->rs_lock);
-        cfs_spin_unlock(&svc->srv_rs_lock);
-        EXIT;
-}
-
-int target_handle_qc_callback(struct ptlrpc_request *req)
-{
-        struct obd_quotactl *oqctl;
-        struct client_obd *cli = &req->rq_export->exp_obd->u.cli;
-
-        oqctl = req_capsule_client_get(&req->rq_pill, &RMF_OBD_QUOTACTL);
-        if (oqctl == NULL) {
-                CERROR("Can't unpack obd_quotactl\n");
-                RETURN(-EPROTO);
-        }
-
-        cli->cl_qchk_stat = oqctl->qc_stat;
-
-        return 0;
-}
-
-#ifdef HAVE_QUOTA_SUPPORT
-int target_handle_dqacq_callback(struct ptlrpc_request *req)
-{
-#ifdef __KERNEL__
-        struct obd_device *obd = req->rq_export->exp_obd;
-        struct obd_device *master_obd = NULL, *lov_obd = NULL;
-        struct obd_device_target *obt;
-        struct lustre_quota_ctxt *qctxt;
-        struct qunit_data *qdata = NULL;
-        int rc = 0;
-        ENTRY;
-
-        if (OBD_FAIL_CHECK(OBD_FAIL_MDS_DROP_QUOTA_REQ))
-                RETURN(rc);
-
-        rc = req_capsule_server_pack(&req->rq_pill);
-        if (rc) {
-                CERROR("packing reply failed!: rc = %d\n", rc);
-                RETURN(rc);
-        }
-
-        LASSERT(req->rq_export);
-
-        qdata = quota_get_qdata(req, QUOTA_REQUEST, QUOTA_EXPORT);
-        if (IS_ERR(qdata)) {
-                rc = PTR_ERR(qdata);
-                CDEBUG(D_ERROR, "Can't unpack qunit_data(rc: %d)\n", rc);
-                req->rq_status = rc;
-                GOTO(out, rc);
-        }
-
-        /* we use the observer */
-        if (obd_pin_observer(obd, &lov_obd) ||
-            obd_pin_observer(lov_obd, &master_obd)) {
-                CERROR("Can't find the observer, it is recovering\n");
-                req->rq_status = -EAGAIN;
-                GOTO(out, rc);
-        }
-
-        obt = &master_obd->u.obt;
-        qctxt = &obt->obt_qctxt;
-
-        if (!qctxt->lqc_setup || !qctxt->lqc_valid) {
-                /* quota_type has not been processed yet, return EAGAIN
-                 * until we know whether or not quotas are supposed to
-                 * be enabled */
-                CDEBUG(D_QUOTA, "quota_type not processed yet, return "
-                       "-EAGAIN\n");
-                req->rq_status = -EAGAIN;
-                GOTO(out, rc);
-        }
-
-        cfs_down_read(&obt->obt_rwsem);
-        if (qctxt->lqc_lqs_hash == NULL) {
-                cfs_up_read(&obt->obt_rwsem);
-                /* quota_type has not been processed yet, return EAGAIN
-                 * until we know whether or not quotas are supposed to
-                 * be enabled */
-                CDEBUG(D_QUOTA, "quota_ctxt is not ready yet, return "
-                       "-EAGAIN\n");
-                req->rq_status = -EAGAIN;
-                GOTO(out, rc);
-        }
-
-        LASSERT(qctxt->lqc_handler);
-        rc = qctxt->lqc_handler(master_obd, qdata,
-                                lustre_msg_get_opc(req->rq_reqmsg));
-        cfs_up_read(&obt->obt_rwsem);
-        if (rc && rc != -EDQUOT)
-                CDEBUG(rc == -EBUSY  ? D_QUOTA : D_ERROR,
-                       "dqacq/dqrel failed! (rc:%d)\n", rc);
-        req->rq_status = rc;
-
-        rc = quota_copy_qdata(req, qdata, QUOTA_REPLY, QUOTA_EXPORT);
-        if (rc < 0) {
-                CERROR("Can't pack qunit_data(rc: %d)\n", rc);
-                GOTO(out, rc);
-        }
-
-        /* Block the quota req. b=14840 */
-        OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_BLOCK_QUOTA_REQ, obd_timeout);
-        EXIT;
-
-out:
-        if (master_obd)
-                obd_unpin_observer(lov_obd);
-        if (lov_obd)
-                obd_unpin_observer(obd);
-
-        rc = ptlrpc_reply(req);
-        return rc;
-#else
-        return 0;
-#endif /* !__KERNEL__ */
+               cfs_list_add(&rs->rs_list, &svcpt->scp_rep_active);
+               rs->rs_scheduled = 0;   /* allow notifier to schedule */
+       }
+       cfs_spin_unlock(&rs->rs_lock);
+       cfs_spin_unlock(&svcpt->scp_rep_lock);
+       EXIT;
 }
-#endif /* HAVE_QUOTA_SUPPORT */
 
 ldlm_mode_t lck_compat_array[] = {
         [LCK_EX] LCK_COMPAT_EX,
@@ -2617,7 +2607,7 @@ void ldlm_dump_export_locks(struct obd_export *exp)
             CERROR("dumping locks for export %p,"
                    "ignore if the unmount doesn't hang\n", exp);
             cfs_list_for_each_entry(lock, &exp->exp_locks_list, l_exp_refs_link)
-                ldlm_lock_dump(D_ERROR, lock, 0);
+                LDLM_ERROR(lock, "lock:");
         }
         cfs_spin_unlock(&exp->exp_locks_list_guard);
 }