lustre/ptlrpc/recovd.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  *  linux/mds/handler.c
   5  *
   6  *  Lustre High Availability Daemon
   7  *
   8  *  Copyright (C) 2001, 2002 Cluster File Systems, Inc.
   9  *
  10  *  This code is issued under the GNU General Public License.
  11  *  See the file COPYING in this distribution
  12  *
  13  *  by Peter Braam <braam@clusterfs.com>
  14  *
  15  */
  16
  17 #define EXPORT_SYMTAB
  18 #define DEBUG_SUBSYSTEM S_RPC
  19
  20 #include <linux/kmod.h>
  21 #include <linux/lustre_lite.h>
  22 #include <linux/lustre_ha.h>
  23
  24 struct connmgr_obd *ptlrpc_connmgr;
  25
  26 void connmgr_cli_manage(struct connmgr_obd *mgr, struct ptlrpc_client *cli)
  27 {
  28         ENTRY;
  29         cli->cli_ha_mgr = mgr;
  30         spin_lock(&mgr->mgr_lock);
  31         list_add(&cli->cli_ha_item, &mgr->mgr_connections_lh);
  32         spin_unlock(&mgr->mgr_lock);
  33         EXIT;
  34 }
  35
  36 void connmgr_cli_fail(struct ptlrpc_client *cli)
  37 {
  38         ENTRY;
  39         spin_lock(&cli->cli_ha_mgr->mgr_lock);
  40         cli->cli_ha_mgr->mgr_flags |= SVC_HA_EVENT;
  41         list_del(&cli->cli_ha_item);
  42         list_add(&cli->cli_ha_item, &cli->cli_ha_mgr->mgr_troubled_lh);
  43         spin_unlock(&cli->cli_ha_mgr->mgr_lock);
  44         wake_up(&cli->cli_ha_mgr->mgr_waitq);
  45         EXIT;
  46 }
  47
  48 static int connmgr_upcall(void)
  49 {
  50         char *argv[2];
  51         char *envp[3];
  52
  53         argv[0] = "/usr/src/obd/utils/ha_assist.sh";
  54         argv[1] = NULL;
  55
  56         envp [0] = "HOME=/";
  57         envp [1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
  58         envp [2] = NULL;
  59
  60         return call_usermodehelper(argv[0], argv, envp);
  61 }
  62
  63 static void connmgr_unpack_body(struct ptlrpc_request *req)
  64 {
  65         struct connmgr_body *b = lustre_msg_buf(req->rq_repmsg, 0);
  66         if (b == NULL)
  67                 LBUG();
  68
  69         b->generation = NTOH__u32(b->generation);
  70 }
  71
  72 int connmgr_connect(struct connmgr_obd *mgr, struct ptlrpc_connection *conn)
  73 {
  74         struct ptlrpc_request *req;
  75         struct ptlrpc_client *cl;
  76         struct connmgr_body *body;
  77         int rc, size = sizeof(*body);
  78         ENTRY;
  79
  80         if (!mgr) {
  81                 CERROR("no manager\n");
  82                 LBUG();
  83         }
  84         cl = mgr->mgr_client;
  85
  86         req = ptlrpc_prep_req(cl, conn, CONNMGR_CONNECT, 1, &size, NULL);
  87         if (!req)
  88                 GOTO(out, rc = -ENOMEM);
  89
  90         body = lustre_msg_buf(req->rq_reqmsg, 0);
  91         body->generation = HTON__u32(conn->c_generation);
  92         body->conn = (__u64)(unsigned long)conn;
  93         body->conn_token = conn->c_token;
  94
  95         req->rq_replen = lustre_msg_size(1, &size);
  96
  97         rc = ptlrpc_queue_wait(req);
  98         rc = ptlrpc_check_status(req, rc);
  99         if (!rc) {
 100                 connmgr_unpack_body(req);
 101                 body = lustre_msg_buf(req->rq_repmsg, 0);
 102                 CDEBUG(D_NET, "mode: %o\n", body->generation);
 103                 conn->c_remote_conn = body->conn;
 104                 conn->c_remote_token = body->conn_token;
 105         }
 106
 107         ptlrpc_free_req(req);
 108         EXIT;
 109  out:
 110         return rc;
 111 }
 112
 113 static int connmgr_handle_connect(struct ptlrpc_request *req)
 114 {
 115         struct connmgr_body *body;
 116         int rc, size = sizeof(*body);
 117         ENTRY;
 118
 119         rc = lustre_pack_msg(1, &size, NULL, &req->rq_replen, &req->rq_repmsg);
 120         if (rc) {
 121                 CERROR("connmgr: out of memory\n");
 122                 req->rq_status = -ENOMEM;
 123                 RETURN(0);
 124         }
 125
 126         body = lustre_msg_buf(req->rq_reqmsg, 0);
 127         connmgr_unpack_body(req);
 128
 129         req->rq_connection->c_remote_conn = body->conn;
 130         req->rq_connection->c_remote_token = body->conn_token;
 131
 132         CERROR("incoming generation %d\n", body->generation);
 133         body = lustre_msg_buf(req->rq_repmsg, 0);
 134         body->generation = 4711;
 135         body->conn = (__u64)(unsigned long)req->rq_connection;
 136         body->conn_token = req->rq_connection->c_token;
 137
 138         RETURN(0);
 139 }
 140
 141 int connmgr_handle(struct obd_device *dev, struct ptlrpc_service *svc,
 142                    struct ptlrpc_request *req)
 143 {
 144         int rc;
 145         ENTRY;
 146
 147         rc = lustre_unpack_msg(req->rq_reqmsg, req->rq_reqlen);
 148         if (rc) {
 149                 CERROR("Invalid request\n");
 150                 GOTO(out, rc);
 151         }
 152
 153         if (req->rq_reqmsg->type != PTL_RPC_REQUEST) {
 154                 CERROR("wrong packet type sent %d\n",
 155                        req->rq_reqmsg->type);
 156                 GOTO(out, rc = -EINVAL);
 157         }
 158
 159         switch (req->rq_reqmsg->opc) {
 160         case CONNMGR_CONNECT:
 161                 CDEBUG(D_INODE, "connect\n");
 162                 OBD_FAIL_RETURN(OBD_FAIL_MDS_GETATTR_NET, 0);
 163                 rc = connmgr_handle_connect(req);
 164                 break;
 165
 166         default:
 167                 rc = ptlrpc_error(svc, req);
 168                 RETURN(rc);
 169         }
 170
 171         EXIT;
 172 out:
 173         if (rc) {
 174                 ptlrpc_error(svc, req);
 175         } else {
 176                 CDEBUG(D_NET, "sending reply\n");
 177                 ptlrpc_reply(svc, req);
 178         }
 179
 180         return 0;
 181 }
 182
 183 static int recovd_check_event(struct connmgr_obd *mgr)
 184 {
 185         int rc = 0;
 186         ENTRY;
 187
 188         spin_lock(&mgr->mgr_lock);
 189
 190         if (!(mgr->mgr_flags & MGR_WORKING) &&
 191             !list_empty(&mgr->mgr_troubled_lh)) {
 192
 193                 CERROR("connection in trouble - state: WORKING, upcall\n");
 194                 mgr->mgr_flags = MGR_WORKING;
 195
 196                 mgr->mgr_waketime = CURRENT_TIME;
 197                 mgr->mgr_timeout = 5 * HZ;
 198                 schedule_timeout(mgr->mgr_timeout);
 199
 200         }
 201
 202         if (mgr->mgr_flags & MGR_WORKING &&
 203             CURRENT_TIME <= mgr->mgr_waketime + mgr->mgr_timeout) {
 204                 CERROR("WORKING: new event\n");
 205
 206                 mgr->mgr_waketime = CURRENT_TIME;
 207                 schedule_timeout(mgr->mgr_timeout);
 208         }
 209
 210         if (mgr->mgr_flags & MGR_STOPPING) {
 211                 CERROR("ha mgr stopping\n");
 212                 rc = 1;
 213         }
 214
 215         spin_unlock(&mgr->mgr_lock);
 216         RETURN(rc);
 217 }
 218
 219 static int recovd_handle_event(struct connmgr_obd *mgr)
 220 {
 221         spin_lock(&mgr->mgr_lock);
 222
 223         if (!(mgr->mgr_flags & MGR_WORKING) &&
 224             !list_empty(&mgr->mgr_troubled_lh)) {
 225
 226                 CERROR("connection in trouble - state: WORKING, upcall\n");
 227                 mgr->mgr_flags = MGR_WORKING;
 228
 229
 230                 connmgr_upcall();
 231                 mgr->mgr_waketime = CURRENT_TIME;
 232                 mgr->mgr_timeout = 5 * HZ;
 233                 schedule_timeout(mgr->mgr_timeout);
 234         }
 235
 236         if (mgr->mgr_flags & MGR_WORKING &&
 237             CURRENT_TIME <= mgr->mgr_waketime + mgr->mgr_timeout) {
 238                 CERROR("WORKING: new event\n");
 239
 240                 mgr->mgr_waketime = CURRENT_TIME;
 241                 schedule_timeout(mgr->mgr_timeout);
 242         }
 243
 244         spin_unlock(&mgr->mgr_lock);
 245         return 0;
 246 }
 247
 248 static int recovd_main(void *arg)
 249 {
 250         struct connmgr_thread *data = (struct connmgr_thread *)arg;
 251         struct connmgr_obd *mgr = data->mgr;
 252
 253         ENTRY;
 254
 255         lock_kernel();
 256         daemonize();
 257         spin_lock_irq(&current->sigmask_lock);
 258         sigfillset(&current->blocked);
 259         recalc_sigpending(current);
 260         spin_unlock_irq(&current->sigmask_lock);
 261
 262         sprintf(current->comm, data->name);
 263
 264         /* Record that the  thread is running */
 265         mgr->mgr_thread = current;
 266         mgr->mgr_flags = MGR_RUNNING;
 267         wake_up(&mgr->mgr_ctl_waitq);
 268
 269         /* And now, loop forever on requests */
 270         while (1) {
 271                 wait_event_interruptible(mgr->mgr_waitq,
 272                                          recovd_check_event(mgr));
 273
 274                 spin_lock(&mgr->mgr_lock);
 275                 if (mgr->mgr_flags & MGR_STOPPING) {
 276                         spin_unlock(&mgr->mgr_lock);
 277                         CERROR("lustre_hamgr quitting\n");
 278                         EXIT;
 279                         break;
 280                 }
 281
 282                 recovd_handle_event(mgr);
 283                 spin_unlock(&mgr->mgr_lock);
 284         }
 285
 286         mgr->mgr_thread = NULL;
 287         mgr->mgr_flags = MGR_STOPPED;
 288         wake_up(&mgr->mgr_ctl_waitq);
 289         CDEBUG(D_NET, "mgr exiting process %d\n", current->pid);
 290         RETURN(0);
 291 }
 292
 293 int recovd_setup(struct connmgr_obd *mgr)
 294 {
 295         struct connmgr_thread d;
 296         int rc;
 297         ENTRY;
 298
 299         INIT_LIST_HEAD(&mgr->mgr_connections_lh);
 300         INIT_LIST_HEAD(&mgr->mgr_troubled_lh);
 301         spin_lock_init(&mgr->mgr_lock);
 302
 303         d.mgr = mgr;
 304         d.name = "lustre_recovd";
 305
 306         init_waitqueue_head(&mgr->mgr_waitq);
 307         init_waitqueue_head(&mgr->mgr_ctl_waitq);
 308
 309         rc = kernel_thread(recovd_main, (void *) &d,
 310                            CLONE_VM | CLONE_FS | CLONE_FILES);
 311         if (rc < 0) {
 312                 CERROR("cannot start thread\n");
 313                 RETURN(-EINVAL);
 314         }
 315         wait_event(mgr->mgr_ctl_waitq, mgr->mgr_flags & MGR_RUNNING);
 316
 317         RETURN(0);
 318 }
 319
 320 int recovd_cleanup(struct connmgr_obd *mgr)
 321 {
 322         mgr->mgr_flags = MGR_STOPPING;
 323
 324         wake_up(&mgr->mgr_waitq);
 325         wait_event_interruptible(mgr->mgr_ctl_waitq,
 326                                  (mgr->mgr_flags & MGR_STOPPED));
 327         RETURN(0);
 328 }