lustre/ptlrpc/recovd.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  *  obd/rpc/recovd.c
   5  *
   6  *  Lustre High Availability Daemon
   7  *
   8  *  Copyright (C) 2001, 2002 Cluster File Systems, Inc.
   9  *
  10  *  This code is issued under the GNU General Public License.
  11  *  See the file COPYING in this distribution
  12  *
  13  *  by Peter Braam <braam@clusterfs.com>
  14  *
  15  */
  16
  17 #define EXPORT_SYMTAB
  18 #define DEBUG_SUBSYSTEM S_RPC
  19
  20 #include <linux/kmod.h>
  21 #include <linux/lustre_lite.h>
  22 #include <linux/lustre_ha.h>
  23
  24 struct recovd_obd *ptlrpc_connmgr;
  25
  26 void connmgr_cli_manage(struct recovd_obd *recovd, struct ptlrpc_client *cli)
  27 {
  28         ENTRY;
  29         cli->cli_recovd = recovd;
  30         spin_lock(&recovd->recovd_lock);
  31         list_add(&cli->cli_ha_item, &recovd->recovd_connections_lh);
  32         spin_unlock(&recovd->recovd_lock);
  33         EXIT;
  34 }
  35
  36 void connmgr_cli_fail(struct ptlrpc_client *cli)
  37 {
  38         ENTRY;
  39         spin_lock(&cli->cli_recovd->recovd_lock);
  40         cli->cli_recovd->recovd_flags |= SVC_HA_EVENT;
  41         list_del(&cli->cli_ha_item);
  42         list_add(&cli->cli_ha_item, &cli->cli_recovd->recovd_troubled_lh);
  43         spin_unlock(&cli->cli_recovd->recovd_lock);
  44         wake_up(&cli->cli_recovd->recovd_waitq);
  45         EXIT;
  46 }
  47
  48 static int connmgr_upcall(void)
  49 {
  50         char *argv[2];
  51         char *envp[3];
  52
  53         argv[0] = "/usr/src/obd/utils/ha_assist.sh";
  54         argv[1] = NULL;
  55
  56         envp [0] = "HOME=/";
  57         envp [1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
  58         envp [2] = NULL;
  59
  60         return call_usermodehelper(argv[0], argv, envp);
  61 }
  62
  63 static void connmgr_unpack_body(struct ptlrpc_request *req)
  64 {
  65         struct connmgr_body *b = lustre_msg_buf(req->rq_repmsg, 0);
  66         if (b == NULL)
  67                 LBUG();
  68
  69         b->generation = NTOH__u32(b->generation);
  70 }
  71
  72 int connmgr_connect(struct recovd_obd *recovd, struct ptlrpc_connection *conn)
  73 {
  74         struct ptlrpc_request *req;
  75         struct ptlrpc_client *cl;
  76         struct connmgr_body *body;
  77         int rc, size = sizeof(*body);
  78         ENTRY;
  79
  80         if (!recovd) {
  81                 CERROR("no manager\n");
  82                 LBUG();
  83         }
  84         cl = recovd->recovd_client;
  85
  86         req = ptlrpc_prep_req(cl, conn, CONNMGR_CONNECT, 1, &size, NULL);
  87         if (!req)
  88                 GOTO(out, rc = -ENOMEM);
  89
  90         body = lustre_msg_buf(req->rq_reqmsg, 0);
  91         body->generation = HTON__u32(conn->c_generation);
  92         body->conn = (__u64)(unsigned long)conn;
  93         body->conn_token = conn->c_token;
  94
  95         req->rq_replen = lustre_msg_size(1, &size);
  96
  97         rc = ptlrpc_queue_wait(req);
  98         rc = ptlrpc_check_status(req, rc);
  99         if (!rc) {
 100                 connmgr_unpack_body(req);
 101                 body = lustre_msg_buf(req->rq_repmsg, 0);
 102                 CDEBUG(D_NET, "remote generation: %o\n", body->generation);
 103                 conn->c_level = LUSTRE_CONN_CON;
 104                 conn->c_remote_conn = body->conn;
 105                 conn->c_remote_token = body->conn_token;
 106         }
 107
 108         ptlrpc_free_req(req);
 109         EXIT;
 110  out:
 111         return rc;
 112 }
 113
 114 static int connmgr_handle_connect(struct ptlrpc_request *req)
 115 {
 116         struct connmgr_body *body;
 117         int rc, size = sizeof(*body);
 118         ENTRY;
 119
 120         rc = lustre_pack_msg(1, &size, NULL, &req->rq_replen, &req->rq_repmsg);
 121         if (rc) {
 122                 CERROR("connmgr: out of memory\n");
 123                 req->rq_status = -ENOMEM;
 124                 RETURN(0);
 125         }
 126
 127         body = lustre_msg_buf(req->rq_reqmsg, 0);
 128         connmgr_unpack_body(req);
 129
 130         req->rq_connection->c_remote_conn = body->conn;
 131         req->rq_connection->c_remote_token = body->conn_token;
 132
 133         CERROR("incoming generation %d\n", body->generation);
 134         body = lustre_msg_buf(req->rq_repmsg, 0);
 135         body->generation = 4711;
 136         body->conn = (__u64)(unsigned long)req->rq_connection;
 137         body->conn_token = req->rq_connection->c_token;
 138
 139         req->rq_connection->c_level = LUSTRE_CONN_CON;
 140         RETURN(0);
 141 }
 142
 143 int connmgr_handle(struct obd_device *dev, struct ptlrpc_service *svc,
 144                    struct ptlrpc_request *req)
 145 {
 146         int rc;
 147         ENTRY;
 148
 149         rc = lustre_unpack_msg(req->rq_reqmsg, req->rq_reqlen);
 150         if (rc) {
 151                 CERROR("Invalid request\n");
 152                 GOTO(out, rc);
 153         }
 154
 155         if (req->rq_reqmsg->type != NTOH__u32(PTL_RPC_MSG_REQUEST)) {
 156                 CERROR("wrong packet type sent %d\n",
 157                        req->rq_reqmsg->type);
 158                 GOTO(out, rc = -EINVAL);
 159         }
 160
 161         switch (req->rq_reqmsg->opc) {
 162         case CONNMGR_CONNECT:
 163                 CDEBUG(D_INODE, "connmgr connect\n");
 164                 rc = connmgr_handle_connect(req);
 165                 break;
 166
 167         default:
 168                 rc = ptlrpc_error(svc, req);
 169                 RETURN(rc);
 170         }
 171
 172         EXIT;
 173 out:
 174         if (rc) {
 175                 ptlrpc_error(svc, req);
 176         } else {
 177                 CDEBUG(D_NET, "sending reply\n");
 178                 ptlrpc_reply(svc, req);
 179         }
 180
 181         return 0;
 182 }
 183
 184 static int recovd_check_event(struct recovd_obd *recovd)
 185 {
 186         int rc = 0;
 187         ENTRY;
 188
 189         spin_lock(&recovd->recovd_lock);
 190
 191         if (!(recovd->recovd_flags & MGR_WORKING) &&
 192             !list_empty(&recovd->recovd_troubled_lh)) {
 193
 194                 CERROR("connection in trouble - state: WORKING, upcall\n");
 195                 recovd->recovd_flags = MGR_WORKING;
 196
 197                 recovd->recovd_waketime = CURRENT_TIME;
 198                 recovd->recovd_timeout = 5 * HZ;
 199                 schedule_timeout(recovd->recovd_timeout);
 200         }
 201
 202         if (recovd->recovd_flags & MGR_WORKING &&
 203             CURRENT_TIME <= recovd->recovd_waketime + recovd->recovd_timeout) {
 204                 CERROR("WORKING: new event\n");
 205
 206                 recovd->recovd_waketime = CURRENT_TIME;
 207                 schedule_timeout(recovd->recovd_timeout);
 208         }
 209
 210         if (recovd->recovd_flags & MGR_STOPPING) {
 211                 CERROR("ha mgr stopping\n");
 212                 rc = 1;
 213         }
 214
 215         spin_unlock(&recovd->recovd_lock);
 216         RETURN(rc);
 217 }
 218
 219 static int recovd_handle_event(struct recovd_obd *recovd)
 220 {
 221         spin_lock(&recovd->recovd_lock);
 222
 223         if (!(recovd->recovd_flags & MGR_WORKING) &&
 224             !list_empty(&recovd->recovd_troubled_lh)) {
 225
 226                 CERROR("connection in trouble - state: WORKING, upcall\n");
 227                 recovd->recovd_flags = MGR_WORKING;
 228
 229
 230                 connmgr_upcall();
 231                 recovd->recovd_waketime = CURRENT_TIME;
 232                 recovd->recovd_timeout = 5 * HZ;
 233                 schedule_timeout(recovd->recovd_timeout);
 234         }
 235
 236         if (recovd->recovd_flags & MGR_WORKING &&
 237             CURRENT_TIME <= recovd->recovd_waketime + recovd->recovd_timeout) {
 238                 CERROR("WORKING: new event\n");
 239
 240                 recovd->recovd_waketime = CURRENT_TIME;
 241                 schedule_timeout(recovd->recovd_timeout);
 242         }
 243
 244         spin_unlock(&recovd->recovd_lock);
 245         return 0;
 246 }
 247
 248 static int recovd_main(void *arg)
 249 {
 250         struct recovd_obd *recovd = (struct recovd_obd *)arg;
 251
 252         ENTRY;
 253
 254         lock_kernel();
 255         daemonize();
 256         spin_lock_irq(&current->sigmask_lock);
 257         sigfillset(&current->blocked);
 258         recalc_sigpending(current);
 259         spin_unlock_irq(&current->sigmask_lock);
 260
 261         sprintf(current->comm, "lustre_recovd");
 262
 263         /* Record that the  thread is running */
 264         recovd->recovd_thread = current;
 265         recovd->recovd_flags = MGR_RUNNING;
 266         wake_up(&recovd->recovd_ctl_waitq);
 267
 268         /* And now, loop forever on requests */
 269         while (1) {
 270                 wait_event_interruptible(recovd->recovd_waitq,
 271                                          recovd_check_event(recovd));
 272
 273                 spin_lock(&recovd->recovd_lock);
 274                 if (recovd->recovd_flags & MGR_STOPPING) {
 275                         spin_unlock(&recovd->recovd_lock);
 276                         CERROR("lustre_hamgr quitting\n");
 277                         EXIT;
 278                         break;
 279                 }
 280
 281                 recovd_handle_event(recovd);
 282                 spin_unlock(&recovd->recovd_lock);
 283         }
 284
 285         recovd->recovd_thread = NULL;
 286         recovd->recovd_flags = MGR_STOPPED;
 287         wake_up(&recovd->recovd_ctl_waitq);
 288         CDEBUG(D_NET, "mgr exiting process %d\n", current->pid);
 289         RETURN(0);
 290 }
 291
 292 int recovd_setup(struct recovd_obd *recovd)
 293 {
 294         int rc;
 295         ENTRY;
 296
 297         INIT_LIST_HEAD(&recovd->recovd_connections_lh);
 298         INIT_LIST_HEAD(&recovd->recovd_troubled_lh);
 299         spin_lock_init(&recovd->recovd_lock);
 300
 301         init_waitqueue_head(&recovd->recovd_waitq);
 302         init_waitqueue_head(&recovd->recovd_recovery_waitq);
 303         init_waitqueue_head(&recovd->recovd_ctl_waitq);
 304
 305         rc = kernel_thread(recovd_main, (void *)recovd,
 306                            CLONE_VM | CLONE_FS | CLONE_FILES);
 307         if (rc < 0) {
 308                 CERROR("cannot start thread\n");
 309                 RETURN(-EINVAL);
 310         }
 311         wait_event(recovd->recovd_ctl_waitq, recovd->recovd_flags & MGR_RUNNING);
 312
 313         RETURN(0);
 314 }
 315
 316 int recovd_cleanup(struct recovd_obd *recovd)
 317 {
 318         recovd->recovd_flags = MGR_STOPPING;
 319
 320         wake_up(&recovd->recovd_waitq);
 321         wait_event_interruptible(recovd->recovd_ctl_waitq,
 322                                  (recovd->recovd_flags & MGR_STOPPED));
 323         RETURN(0);
 324 }