lustre/ptlrpc/recover.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.gnu.org/licenses/gpl-2.0.html
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Use is subject to license terms.
  25  *
  26  * Copyright (c) 2011, 2017, Intel Corporation.
  27  */
  28 /*
  29  * This file is part of Lustre, http://www.lustre.org/
  30  * Lustre is a trademark of Sun Microsystems, Inc.
  31  *
  32  * lustre/ptlrpc/recover.c
  33  *
  34  * Author: Mike Shaver <shaver@clusterfs.com>
  35  */
  36
  37 #define DEBUG_SUBSYSTEM S_RPC
  38 #include <linux/list.h>
  39 #include <libcfs/libcfs.h>
  40 #include <obd_support.h>
  41 #include <lustre_ha.h>
  42 #include <lustre_net.h>
  43 #include <lustre_import.h>
  44 #include <lustre_export.h>
  45 #include <obd.h>
  46 #include <obd_class.h>
  47
  48 #include "ptlrpc_internal.h"
  49
  50 /**
  51  * Start recovery on disconnected import.
  52  * This is done by just attempting a connect
  53  */
  54 void ptlrpc_initiate_recovery(struct obd_import *imp)
  55 {
  56         ENTRY;
  57
  58         CDEBUG(D_HA, "%s: starting recovery\n", obd2cli_tgt(imp->imp_obd));
  59         ptlrpc_connect_import(imp);
  60
  61         EXIT;
  62 }
  63
  64 /**
  65  * Identify what request from replay list needs to be replayed next
  66  * (based on what we have already replayed) and send it to server.
  67  */
  68 int ptlrpc_replay_next(struct obd_import *imp, int *inflight)
  69 {
  70         int rc = 0;
  71         struct ptlrpc_request *req = NULL;
  72         __u64 last_transno;
  73         ENTRY;
  74
  75         *inflight = 0;
  76
  77         /* It might have committed some after we last spoke, so make sure we
  78          * get rid of them now.
  79          */
  80         spin_lock(&imp->imp_lock);
  81         imp->imp_last_transno_checked = 0;
  82         ptlrpc_free_committed(imp);
  83         last_transno = imp->imp_last_replay_transno;
  84
  85         CDEBUG(D_HA, "import %p from %s committed %llu last %llu\n",
  86                imp, obd2cli_tgt(imp->imp_obd),
  87                imp->imp_peer_committed_transno, last_transno);
  88
  89         /* Replay all the committed open requests on committed_list first */
  90         if (!list_empty(&imp->imp_committed_list)) {
  91                 req = list_last_entry(&imp->imp_committed_list,
  92                                       struct ptlrpc_request, rq_replay_list);
  93
  94                 /* The last request on committed_list hasn't been replayed */
  95                 if (req->rq_transno > last_transno) {
  96                         if (!imp->imp_resend_replay ||
  97                             imp->imp_replay_cursor == &imp->imp_committed_list)
  98                                 imp->imp_replay_cursor =
  99                                         imp->imp_replay_cursor->next;
 100
 101                         while (imp->imp_replay_cursor !=
 102                                &imp->imp_committed_list) {
 103                                 req = list_entry(imp->imp_replay_cursor,
 104                                                  struct ptlrpc_request,
 105                                                  rq_replay_list);
 106                                 if (req->rq_transno > last_transno)
 107                                         break;
 108
 109                                 req = NULL;
 110                                 LASSERT(!list_empty(imp->imp_replay_cursor));
 111                                 imp->imp_replay_cursor =
 112                                         imp->imp_replay_cursor->next;
 113                         }
 114                 } else {
 115                         /* All requests on committed_list have been replayed */
 116                         imp->imp_replay_cursor = &imp->imp_committed_list;
 117                         req = NULL;
 118                 }
 119         }
 120
 121         /* All the requests in committed list have been replayed, let's replay
 122          * the imp_replay_list */
 123         if (req == NULL) {
 124                 struct ptlrpc_request *tmp;
 125
 126                 list_for_each_entry(tmp, &imp->imp_replay_list,
 127                                     rq_replay_list) {
 128                         if (tmp->rq_transno > last_transno) {
 129                                 req = tmp;
 130                                 break;
 131                         }
 132                 }
 133         }
 134
 135         /* If need to resend the last sent transno (because a reconnect
 136          * has occurred), then stop on the matching req and send it again.
 137          * If, however, the last sent transno has been committed then we
 138          * continue replay from the next request. */
 139         if (req != NULL && imp->imp_resend_replay)
 140                 lustre_msg_add_flags(req->rq_reqmsg, MSG_RESENT);
 141
 142         /* ptlrpc_prepare_replay() may fail to add the reqeust into unreplied
 143          * list if the request hasn't been added to replay list then. Another
 144          * exception is that resend replay could have been removed from the
 145          * unreplied list. */
 146         if (req != NULL && list_empty(&req->rq_unreplied_list)) {
 147                 DEBUG_REQ(D_HA, req, "resend_replay=%d, last_transno=%llu",
 148                           imp->imp_resend_replay, last_transno);
 149                 ptlrpc_add_unreplied(req);
 150                 imp->imp_known_replied_xid = ptlrpc_known_replied_xid(imp);
 151         }
 152
 153         imp->imp_resend_replay = 0;
 154         spin_unlock(&imp->imp_lock);
 155
 156         if (req != NULL) {
 157                 LASSERT(!list_empty(&req->rq_unreplied_list));
 158
 159                 rc = ptlrpc_replay_req(req);
 160                 if (rc) {
 161                         CERROR("recovery replay error %d for req %llu\n",
 162                                rc, req->rq_xid);
 163                         RETURN(rc);
 164                 }
 165                 *inflight = 1;
 166         }
 167         RETURN(rc);
 168 }
 169
 170 /**
 171  * Schedule resending of request on sending_list. This is done after
 172  * we completed replaying of requests and locks.
 173  */
 174 int ptlrpc_resend(struct obd_import *imp)
 175 {
 176         struct ptlrpc_request *req;
 177
 178         ENTRY;
 179
 180         /* As long as we're in recovery, nothing should be added to the sending
 181          * list, so we don't need to hold the lock during this iteration and
 182          * resend process.
 183          */
 184         /* Well... what if lctl recover is called twice at the same time?
 185          */
 186         spin_lock(&imp->imp_lock);
 187         if (imp->imp_state != LUSTRE_IMP_RECOVER) {
 188                 spin_unlock(&imp->imp_lock);
 189                 RETURN(-1);
 190         }
 191
 192         list_for_each_entry(req, &imp->imp_sending_list, rq_list) {
 193                 LASSERTF((long)req > PAGE_SIZE && req != LP_POISON,
 194                          "req %p bad\n", req);
 195                 LASSERTF(req->rq_type != LI_POISON, "req %p freed\n", req);
 196
 197                 /* If the request is allowed to be sent during replay and it
 198                  * is not timeout yet, then it does not need to be resent. */
 199                 if (!ptlrpc_no_resend(req) &&
 200                     (req->rq_timedout || !req->rq_allow_replay))
 201                         ptlrpc_resend_req(req);
 202         }
 203         spin_unlock(&imp->imp_lock);
 204
 205         OBD_FAIL_TIMEOUT(OBD_FAIL_LDLM_ENQUEUE_OLD_EXPORT, 2);
 206         RETURN(0);
 207 }
 208
 209 /**
 210  * Go through all requests in delayed list and wake their threads
 211  * for resending
 212  */
 213 void ptlrpc_wake_delayed(struct obd_import *imp)
 214 {
 215         struct ptlrpc_request *req;
 216
 217         spin_lock(&imp->imp_lock);
 218         list_for_each_entry(req, &imp->imp_delayed_list, rq_list) {
 219                 DEBUG_REQ(D_HA, req, "waking (set %p):", req->rq_set);
 220                 ptlrpc_client_wake_req(req);
 221         }
 222         spin_unlock(&imp->imp_lock);
 223 }
 224
 225 void ptlrpc_request_handle_notconn(struct ptlrpc_request *failed_req)
 226 {
 227         struct obd_import *imp = failed_req->rq_import;
 228         int conn = lustre_msg_get_conn_cnt(failed_req->rq_reqmsg);
 229         ENTRY;
 230
 231         CDEBUG(D_HA, "import %s of %s@%s abruptly disconnected: reconnecting\n",
 232                 imp->imp_obd->obd_name, obd2cli_tgt(imp->imp_obd),
 233                 imp->imp_connection->c_remote_uuid.uuid);
 234
 235         if (ptlrpc_set_import_discon(imp, conn, true)) {
 236                 /* to control recovery via lctl {disable|enable}_recovery */
 237                 if (imp->imp_deactive == 0)
 238                         ptlrpc_connect_import(imp);
 239         }
 240
 241         /* Wait for recovery to complete and resend. If evicted, then
 242            this request will be errored out later.*/
 243         spin_lock(&failed_req->rq_lock);
 244         if (!failed_req->rq_no_resend)
 245                 failed_req->rq_resend = 1;
 246         spin_unlock(&failed_req->rq_lock);
 247
 248         EXIT;
 249 }
 250
 251 /**
 252  * Administratively active/deactive a client.
 253  * This should only be called by the ioctl interface, currently
 254  *  - the lctl deactivate and activate commands
 255  *  - echo 0/1 >> /proc/osc/XXX/active
 256  *  - client umount -f (ll_umount_begin)
 257  */
 258 int ptlrpc_set_import_active(struct obd_import *imp, int active)
 259 {
 260         struct obd_device *obd = imp->imp_obd;
 261         int rc = 0;
 262
 263         ENTRY;
 264         LASSERT(obd);
 265
 266         /* When deactivating, mark import invalid, and abort in-flight
 267          * requests. */
 268         if (!active) {
 269                 LCONSOLE_WARN("setting import %s INACTIVE by administrator "
 270                               "request\n", obd2cli_tgt(imp->imp_obd));
 271
 272                 /* set before invalidate to avoid messages about imp_inval
 273                  * set without imp_deactive in ptlrpc_import_delay_req */
 274                 spin_lock(&imp->imp_lock);
 275                 imp->imp_deactive = 1;
 276                 spin_unlock(&imp->imp_lock);
 277
 278                 obd_import_event(imp->imp_obd, imp, IMP_EVENT_DEACTIVATE);
 279
 280                 ptlrpc_invalidate_import(imp);
 281         }
 282
 283         /* When activating, mark import valid, and attempt recovery */
 284         if (active) {
 285                 CDEBUG(D_HA, "setting import %s VALID\n",
 286                        obd2cli_tgt(imp->imp_obd));
 287
 288                 spin_lock(&imp->imp_lock);
 289                 imp->imp_deactive = 0;
 290                 spin_unlock(&imp->imp_lock);
 291                 obd_import_event(imp->imp_obd, imp, IMP_EVENT_ACTIVATE);
 292
 293                 rc = ptlrpc_recover_import(imp, NULL, 0);
 294         }
 295
 296         RETURN(rc);
 297 }
 298 EXPORT_SYMBOL(ptlrpc_set_import_active);
 299
 300 /* Attempt to reconnect an import */
 301 int ptlrpc_recover_import(struct obd_import *imp, char *new_uuid, int async)
 302 {
 303         int rc = 0;
 304         ENTRY;
 305
 306         spin_lock(&imp->imp_lock);
 307         if (imp->imp_state == LUSTRE_IMP_NEW || imp->imp_deactive ||
 308             atomic_read(&imp->imp_inval_count))
 309                 rc = -EINVAL;
 310         spin_unlock(&imp->imp_lock);
 311         if (rc)
 312                 GOTO(out, rc);
 313
 314         /* force import to be disconnected. */
 315         ptlrpc_set_import_discon(imp, 0, false);
 316
 317         if (new_uuid) {
 318                 struct obd_uuid uuid;
 319
 320                 /* intruct import to use new uuid */
 321                 obd_str2uuid(&uuid, new_uuid);
 322                 rc = import_set_conn_priority(imp, &uuid);
 323                 if (rc)
 324                         GOTO(out, rc);
 325         }
 326
 327         /* Check if reconnect is already in progress */
 328         spin_lock(&imp->imp_lock);
 329         if (imp->imp_state != LUSTRE_IMP_DISCON) {
 330                 imp->imp_force_verify = 1;
 331                 rc = -EALREADY;
 332         }
 333         spin_unlock(&imp->imp_lock);
 334         if (rc)
 335                 GOTO(out, rc);
 336
 337         OBD_RACE(OBD_FAIL_PTLRPC_CONNECT_RACE);
 338
 339         rc = ptlrpc_connect_import(imp);
 340         if (rc)
 341                 GOTO(out, rc);
 342
 343         if (!async) {
 344                 long timeout = cfs_time_seconds(obd_timeout);
 345
 346                 CDEBUG(D_HA, "%s: recovery started, waiting %u jiffies\n",
 347                        obd2cli_tgt(imp->imp_obd), obd_timeout);
 348
 349                 rc = wait_event_idle_timeout(imp->imp_recovery_waitq,
 350                                              !ptlrpc_import_in_recovery(imp),
 351                                              timeout);
 352                 if (rc == 0)
 353                         rc = -ETIMEDOUT;
 354                 else
 355                         rc = 0;
 356                 CDEBUG(D_HA, "%s: recovery finished\n",
 357                        obd2cli_tgt(imp->imp_obd));
 358         }
 359         EXIT;
 360
 361 out:
 362         return rc;
 363 }
 364 EXPORT_SYMBOL(ptlrpc_recover_import);
 365
 366 int ptlrpc_import_in_recovery(struct obd_import *imp)
 367 {
 368         int in_recovery = 1;
 369
 370         spin_lock(&imp->imp_lock);
 371         if (imp->imp_state <= LUSTRE_IMP_DISCON ||
 372             imp->imp_state >= LUSTRE_IMP_FULL ||
 373             imp->imp_obd->obd_no_recov)
 374                 in_recovery = 0;
 375         spin_unlock(&imp->imp_lock);
 376
 377         return in_recovery;
 378 }