1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Portal-RPC reconnection and replay operations, for use in recovery.
6 * Copyright (c) 2002, 2003 Cluster File Systems, Inc.
7 * Author: Mike Shaver <shaver@clusterfs.com>
9 * This file is part of the Lustre file system, http://www.lustre.org
10 * Lustre is a trademark of Cluster File Systems, Inc.
12 * You may have signed or agreed to another license before downloading
13 * this software. If so, you are bound by the terms and conditions
14 * of that agreement, and the following does not apply to you. See the
15 * LICENSE file included with this distribution for more information.
17 * If you did not agree to a different license, then this copy of Lustre
18 * is open source software; you can redistribute it and/or modify it
19 * under the terms of version 2 of the GNU General Public License as
20 * published by the Free Software Foundation.
22 * In either case, Lustre is distributed in the hope that it will be
23 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty
24 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 * license text for more details.
28 #define DEBUG_SUBSYSTEM S_RPC
30 # include <libcfs/libcfs.h>
32 # include <liblustre.h>
35 #include <obd_support.h>
36 #include <lustre_ha.h>
37 #include <lustre_net.h>
38 #include <lustre_import.h>
39 #include <lustre_export.h>
42 #include <obd_class.h>
43 #include <obd_lov.h> /* for IOC_LOV_SET_OSC_ACTIVE */
44 #include <libcfs/list.h>
46 #include "ptlrpc_internal.h"
48 static int ptlrpc_recover_import_no_retry(struct obd_import *, char *);
50 void ptlrpc_initiate_recovery(struct obd_import *imp)
54 CDEBUG(D_HA, "%s: starting recovery\n", obd2cli_tgt(imp->imp_obd));
55 ptlrpc_connect_import(imp, NULL);
60 int ptlrpc_replay_next(struct obd_import *imp, int *inflight)
63 struct list_head *tmp, *pos;
64 struct ptlrpc_request *req = NULL;
70 /* It might have committed some after we last spoke, so make sure we
71 * get rid of them now.
73 spin_lock(&imp->imp_lock);
74 imp->imp_last_transno_checked = 0;
75 ptlrpc_free_committed(imp);
76 last_transno = imp->imp_last_replay_transno;
77 spin_unlock(&imp->imp_lock);
79 CDEBUG(D_HA, "import %p from %s committed "LPU64" last "LPU64"\n",
80 imp, obd2cli_tgt(imp->imp_obd),
81 imp->imp_peer_committed_transno, last_transno);
83 /* Do I need to hold a lock across this iteration? We shouldn't be
84 * racing with any additions to the list, because we're in recovery
85 * and are therefore not processing additional requests to add. Calls
86 * to ptlrpc_free_committed might commit requests, but nothing "newer"
87 * than the one we're replaying (it can't be committed until it's
88 * replayed, and we're doing that here). l_f_e_safe protects against
89 * problems with the current request being committed, in the unlikely
90 * event of that race. So, in conclusion, I think that it's safe to
91 * perform this list-walk without the imp_lock held.
93 * But, the {mdc,osc}_replay_open callbacks both iterate
94 * request lists, and have comments saying they assume the
95 * imp_lock is being held by ptlrpc_replay, but it's not. it's
96 * just a little race...
98 list_for_each_safe(tmp, pos, &imp->imp_replay_list) {
99 req = list_entry(tmp, struct ptlrpc_request, rq_replay_list);
101 /* If need to resend the last sent transno (because a
102 reconnect has occurred), then stop on the matching
103 req and send it again. If, however, the last sent
104 transno has been committed then we continue replay
105 from the next request. */
106 if (imp->imp_resend_replay &&
107 req->rq_transno == last_transno) {
108 lustre_msg_add_flags(req->rq_reqmsg, MSG_RESENT);
112 if (req->rq_transno > last_transno) {
113 imp->imp_last_replay_transno = req->rq_transno;
120 imp->imp_resend_replay = 0;
123 rc = ptlrpc_replay_req(req);
125 CERROR("recovery replay error %d for req "
126 LPD64"\n", rc, req->rq_xid);
134 int ptlrpc_resend(struct obd_import *imp)
136 struct ptlrpc_request *req, *next;
140 /* As long as we're in recovery, nothing should be added to the sending
141 * list, so we don't need to hold the lock during this iteration and
144 /* Well... what if lctl recover is called twice at the same time?
146 spin_lock(&imp->imp_lock);
147 if (imp->imp_state != LUSTRE_IMP_RECOVER) {
148 spin_unlock(&imp->imp_lock);
151 spin_unlock(&imp->imp_lock);
153 list_for_each_entry_safe(req, next, &imp->imp_sending_list, rq_list) {
154 LASSERTF((long)req > CFS_PAGE_SIZE && req != LP_POISON,
155 "req %p bad\n", req);
156 LASSERTF(req->rq_type != LI_POISON, "req %p freed\n", req);
157 if (!req->rq_no_resend)
158 ptlrpc_resend_req(req);
164 void ptlrpc_wake_delayed(struct obd_import *imp)
166 struct list_head *tmp, *pos;
167 struct ptlrpc_request *req;
169 spin_lock(&imp->imp_lock);
170 list_for_each_safe(tmp, pos, &imp->imp_delayed_list) {
171 req = list_entry(tmp, struct ptlrpc_request, rq_list);
173 DEBUG_REQ(D_HA, req, "waking (set %p):", req->rq_set);
174 ptlrpc_wake_client_req(req);
176 spin_unlock(&imp->imp_lock);
179 void ptlrpc_request_handle_notconn(struct ptlrpc_request *failed_req)
181 struct obd_import *imp = failed_req->rq_import;
184 CDEBUG(D_HA, "import %s of %s@%s abruptly disconnected: reconnecting\n",
185 imp->imp_obd->obd_name, obd2cli_tgt(imp->imp_obd),
186 imp->imp_connection->c_remote_uuid.uuid);
188 if (ptlrpc_set_import_discon(imp,
189 lustre_msg_get_conn_cnt(failed_req->rq_reqmsg))) {
190 if (!imp->imp_replayable) {
191 CDEBUG(D_HA, "import %s@%s for %s not replayable, "
192 "auto-deactivating\n",
193 obd2cli_tgt(imp->imp_obd),
194 imp->imp_connection->c_remote_uuid.uuid,
195 imp->imp_obd->obd_name);
196 ptlrpc_deactivate_import(imp);
198 /* to control recovery via lctl {disable|enable}_recovery */
199 if (imp->imp_deactive == 0)
200 ptlrpc_connect_import(imp, NULL);
203 /* Wait for recovery to complete and resend. If evicted, then
204 this request will be errored out later.*/
205 spin_lock(&failed_req->rq_lock);
206 if (!failed_req->rq_no_resend)
207 failed_req->rq_resend = 1;
208 spin_unlock(&failed_req->rq_lock);
214 * Administratively active/deactive a client.
215 * This should only be called by the ioctl interface, currently
216 * - the lctl deactivate and activate commands
217 * - echo 0/1 >> /proc/osc/XXX/active
218 * - client umount -f (ll_umount_begin)
220 int ptlrpc_set_import_active(struct obd_import *imp, int active)
222 struct obd_device *obd = imp->imp_obd;
228 /* When deactivating, mark import invalid, and abort in-flight
231 LCONSOLE_WARN("setting import %s INACTIVE by administrator "
232 "request\n", obd2cli_tgt(imp->imp_obd));
233 ptlrpc_invalidate_import(imp);
234 imp->imp_deactive = 1;
237 /* When activating, mark import valid, and attempt recovery */
239 imp->imp_deactive = 0;
240 CDEBUG(D_HA, "setting import %s VALID\n",
241 obd2cli_tgt(imp->imp_obd));
242 rc = ptlrpc_recover_import(imp, NULL);
248 /* Attempt to reconnect an import */
249 int ptlrpc_recover_import(struct obd_import *imp, char *new_uuid)
254 /* force import to be disconnected. */
255 ptlrpc_set_import_discon(imp, 0);
257 imp->imp_deactive = 0;
258 rc = ptlrpc_recover_import_no_retry(imp, new_uuid);
263 int ptlrpc_import_in_recovery(struct obd_import *imp)
266 spin_lock(&imp->imp_lock);
267 if (imp->imp_state == LUSTRE_IMP_FULL ||
268 imp->imp_state == LUSTRE_IMP_CLOSED ||
269 imp->imp_state == LUSTRE_IMP_DISCON)
271 spin_unlock(&imp->imp_lock);
275 static int ptlrpc_recover_import_no_retry(struct obd_import *imp,
280 struct l_wait_info lwi;
283 /* Check if reconnect is already in progress */
284 spin_lock(&imp->imp_lock);
285 if (imp->imp_state != LUSTRE_IMP_DISCON) {
288 spin_unlock(&imp->imp_lock);
290 if (in_recovery == 1)
293 rc = ptlrpc_connect_import(imp, new_uuid);
297 CDEBUG(D_HA, "%s: recovery started, waiting\n",
298 obd2cli_tgt(imp->imp_obd));
300 lwi = LWI_TIMEOUT(cfs_timeout_cap(cfs_time_seconds(obd_timeout)),
302 rc = l_wait_event(imp->imp_recovery_waitq,
303 !ptlrpc_import_in_recovery(imp), &lwi);
304 CDEBUG(D_HA, "%s: recovery finished\n",
305 obd2cli_tgt(imp->imp_obd));