lustre/ptlrpc/recovd.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  *  obd/rpc/recovd.c
   5  *
   6  *  Lustre High Availability Daemon
   7  *
   8  *  Copyright (C) 2001, 2002 Cluster File Systems, Inc.
   9  *
  10  *  This code is issued under the GNU General Public License.
  11  *  See the file COPYING in this distribution
  12  *
  13  *  by Peter Braam <braam@clusterfs.com>
  14  *
  15  */
  16
  17 #define DEBUG_SUBSYSTEM S_RPC
  18
  19 #include <linux/lustre_lite.h>
  20 #include <linux/lustre_ha.h>
  21 #include <linux/obd_support.h>
  22
  23 void recovd_conn_manage(struct ptlrpc_connection *conn,
  24                         struct recovd_obd *recovd, ptlrpc_recovery_cb_t recover)
  25 {
  26         struct recovd_data *rd = &conn->c_recovd_data;
  27         ENTRY;
  28
  29         rd->rd_recovd = recovd;
  30         rd->rd_recover = recover;
  31         rd->rd_phase = RD_IDLE;
  32         rd->rd_next_phase = RD_TROUBLED;
  33
  34         spin_lock(&recovd->recovd_lock);
  35         INIT_LIST_HEAD(&rd->rd_managed_chain);
  36         list_add(&recovd->recovd_managed_items, &rd->rd_managed_chain);
  37         spin_unlock(&recovd->recovd_lock);
  38
  39         EXIT;
  40 }
  41
  42 void recovd_conn_fail(struct ptlrpc_connection *conn)
  43 {
  44         struct recovd_data *rd = &conn->c_recovd_data;
  45         struct recovd_obd *recovd = rd->rd_recovd;
  46         ENTRY;
  47
  48         if (!recovd) {
  49                 CERROR("no recovd for connection %p\n", conn);
  50                 EXIT;
  51                 return;
  52         }
  53
  54
  55         spin_lock(&recovd->recovd_lock);
  56         if (rd->rd_phase != RD_IDLE) {
  57                 CERROR("connection %p to %s already in recovery\n",
  58                        conn, conn->c_remote_uuid);
  59                 /* XXX need to distinguish from failure-in-recovery */
  60                 spin_unlock(&recovd->recovd_lock);
  61                 EXIT;
  62                 return;
  63         }
  64
  65         CERROR("connection %p to %s failed\n", conn, conn->c_remote_uuid);
  66         list_del(&rd->rd_managed_chain);
  67         list_add_tail(&rd->rd_managed_chain, &recovd->recovd_troubled_items);
  68         rd->rd_phase = RD_TROUBLED;
  69         spin_unlock(&recovd->recovd_lock);
  70
  71         wake_up(&recovd->recovd_waitq);
  72
  73         EXIT;
  74 }
  75
  76 void recovd_conn_fixed(struct ptlrpc_connection *conn)
  77 {
  78         struct recovd_data *rd = &conn->c_recovd_data;
  79         ENTRY;
  80
  81         CDEBUG(D_HA, "connection %p (now to %s) fixed\n",
  82                conn, conn->c_remote_uuid);
  83         spin_lock(&rd->rd_recovd->recovd_lock);
  84         list_del(&rd->rd_managed_chain);
  85         rd->rd_phase = RD_IDLE;
  86         rd->rd_next_phase = RD_TROUBLED;
  87         list_add(&rd->rd_managed_chain, &rd->rd_recovd->recovd_managed_items);
  88         spin_unlock(&rd->rd_recovd->recovd_lock);
  89
  90         EXIT;
  91 }
  92
  93
  94 static int recovd_check_event(struct recovd_obd *recovd)
  95 {
  96         int rc = 0;
  97         struct list_head *tmp;
  98
  99         ENTRY;
 100
 101         spin_lock(&recovd->recovd_lock);
 102
 103         if (recovd->recovd_state == RECOVD_STOPPING)
 104                 GOTO(out, rc = 1);
 105
 106         list_for_each(tmp, &recovd->recovd_troubled_items) {
 107
 108                 struct recovd_data *rd = list_entry(tmp, struct recovd_data,
 109                                                     rd_managed_chain);
 110
 111                 if (rd->rd_phase == rd->rd_next_phase ||
 112                     rd->rd_phase == RD_FAILED)
 113                         GOTO(out, rc = 1);
 114         }
 115
 116  out:
 117         spin_unlock(&recovd->recovd_lock);
 118         RETURN(rc);
 119 }
 120
 121 static void dump_connection_list(struct list_head *head)
 122 {
 123         struct list_head *tmp;
 124
 125         list_for_each(tmp, head) {
 126                 struct ptlrpc_connection *conn =
 127                         list_entry(tmp, struct ptlrpc_connection,
 128                                    c_recovd_data.rd_managed_chain);
 129                 CDEBUG(D_HA, "   %p = %s (%d/%d)\n", conn, conn->c_remote_uuid,
 130                        conn->c_recovd_data.rd_phase,
 131                        conn->c_recovd_data.rd_next_phase);
 132         }
 133 }
 134
 135 static int recovd_handle_event(struct recovd_obd *recovd)
 136 {
 137         struct list_head *tmp, *n;
 138         int rc = 0;
 139         ENTRY;
 140
 141         spin_lock(&recovd->recovd_lock);
 142
 143         CERROR("managed: \n");
 144         dump_connection_list(&recovd->recovd_managed_items);
 145         CERROR("troubled: \n");
 146         dump_connection_list(&recovd->recovd_troubled_items);
 147
 148         /*
 149          * We use _safe here because one of the callbacks, expecially
 150          * FAILURE or PREPARED, could move list items around.
 151          */
 152         list_for_each_safe(tmp, n, &recovd->recovd_troubled_items) {
 153                 struct recovd_data *rd = list_entry(tmp, struct recovd_data,
 154                                                     rd_managed_chain);
 155
 156                 if (rd->rd_phase != RD_FAILED &&
 157                     rd->rd_phase != rd->rd_next_phase)
 158                         continue;
 159
 160                 switch (rd->rd_phase) {
 161                     case RD_FAILED:
 162                 cb_failed: /* must always reach here with recovd_lock held! */
 163                         CERROR("recovery FAILED for rd %p (conn %p): %d\n",
 164                                rd, class_rd2conn(rd), rc);
 165
 166                         spin_unlock(&recovd->recovd_lock);
 167                         (void)rd->rd_recover(rd, PTLRPC_RECOVD_PHASE_FAILURE);
 168                         spin_lock(&recovd->recovd_lock);
 169                         break;
 170
 171                     case RD_TROUBLED:
 172                         if (!rd->rd_recover) {
 173                                 CERROR("no rd_recover for rd %p (conn %p)\n",
 174                                        rd, class_rd2conn(rd));
 175                                 rc = -EINVAL;
 176                                 break;
 177                         }
 178                         CERROR("starting recovery for rd %p (conn %p)\n",
 179                                rd, class_rd2conn(rd));
 180                         rd->rd_phase = RD_PREPARING;
 181
 182                         spin_unlock(&recovd->recovd_lock);
 183                         rc = rd->rd_recover(rd, PTLRPC_RECOVD_PHASE_PREPARE);
 184                         spin_lock(&recovd->recovd_lock);
 185                         if (rc)
 186                                 goto cb_failed;
 187
 188                         rd->rd_next_phase = RD_PREPARED;
 189                         break;
 190
 191                     case RD_PREPARED:
 192                         rd->rd_phase = RD_RECOVERING;
 193
 194                         CERROR("recovery prepared for rd %p (conn %p)\n",
 195                                rd, class_rd2conn(rd));
 196
 197                         spin_unlock(&recovd->recovd_lock);
 198                         rc = rd->rd_recover(rd, PTLRPC_RECOVD_PHASE_RECOVER);
 199                         spin_lock(&recovd->recovd_lock);
 200                         if (rc)
 201                                 goto cb_failed;
 202
 203                         rd->rd_next_phase = RD_RECOVERED;
 204                         break;
 205
 206                     case RD_RECOVERED:
 207                         rd->rd_phase = RD_IDLE;
 208                         rd->rd_next_phase = RD_TROUBLED;
 209
 210                         CERROR("recovery complete for rd %p (conn %p)\n",
 211                                rd, class_rd2conn(rd));
 212                         break;
 213
 214                     default:
 215                         break;
 216                 }
 217         }
 218         spin_unlock(&recovd->recovd_lock);
 219         RETURN(0);
 220 }
 221
 222 static int recovd_main(void *arg)
 223 {
 224         struct recovd_obd *recovd = (struct recovd_obd *)arg;
 225
 226         ENTRY;
 227
 228         lock_kernel();
 229         daemonize();
 230         spin_lock_irq(&current->sigmask_lock);
 231         sigfillset(&current->blocked);
 232         recalc_sigpending(current);
 233         spin_unlock_irq(&current->sigmask_lock);
 234
 235         sprintf(current->comm, "lustre_recovd");
 236         unlock_kernel();
 237
 238         /* Signal that the thread is running. */
 239         recovd->recovd_thread = current;
 240         recovd->recovd_state = RECOVD_READY;
 241         wake_up(&recovd->recovd_ctl_waitq);
 242
 243         /* And now, loop forever on requests. */
 244         while (1) {
 245                 wait_event(recovd->recovd_waitq, recovd_check_event(recovd));
 246                 if (recovd->recovd_state == RECOVD_STOPPING)
 247                         break;
 248                 recovd_handle_event(recovd);
 249         }
 250
 251         recovd->recovd_thread = NULL;
 252         recovd->recovd_state = RECOVD_STOPPED;
 253         wake_up(&recovd->recovd_ctl_waitq);
 254         CDEBUG(D_HA, "mgr exiting process %d\n", current->pid);
 255         RETURN(0);
 256 }
 257
 258 int recovd_setup(struct recovd_obd *recovd)
 259 {
 260         int rc;
 261
 262         ENTRY;
 263
 264         INIT_LIST_HEAD(&recovd->recovd_managed_items);
 265         INIT_LIST_HEAD(&recovd->recovd_troubled_items);
 266         spin_lock_init(&recovd->recovd_lock);
 267
 268         init_waitqueue_head(&recovd->recovd_waitq);
 269         init_waitqueue_head(&recovd->recovd_recovery_waitq);
 270         init_waitqueue_head(&recovd->recovd_ctl_waitq);
 271
 272         rc = kernel_thread(recovd_main, (void *)recovd,
 273                            CLONE_VM | CLONE_FS | CLONE_FILES);
 274         if (rc < 0) {
 275                 CERROR("cannot start thread\n");
 276                 RETURN(-EINVAL);
 277         }
 278         wait_event(recovd->recovd_ctl_waitq,
 279                    recovd->recovd_state == RECOVD_READY);
 280
 281         ptlrpc_recovd = recovd;
 282         class_signal_connection_failure = recovd_conn_fail;
 283
 284         RETURN(0);
 285 }
 286
 287 int recovd_cleanup(struct recovd_obd *recovd)
 288 {
 289         ENTRY;
 290         spin_lock(&recovd->recovd_lock);
 291         recovd->recovd_state = RECOVD_STOPPING;
 292         wake_up(&recovd->recovd_waitq);
 293         spin_unlock(&recovd->recovd_lock);
 294
 295         wait_event(recovd->recovd_ctl_waitq,
 296                    (recovd->recovd_state == RECOVD_STOPPED));
 297         RETURN(0);
 298 }
 299
 300 struct recovd_obd *ptlrpc_recovd;