lustre/osp/lwp_dev.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; if not, write to the
  18  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  19  * Boston, MA 021110-1307, USA
  20  *
  21  * GPL HEADER END
  22  */
  23 /*
  24  * Copyright (c) 2013, Intel Corporation.
  25  * Use is subject to license terms.
  26  *
  27  * lustre/osp/lwp_dev.c
  28  *
  29  * Light Weight Proxy, which is just for managing the connection established
  30  * from OSTs/MDTs to MDT0.
  31  *
  32  * Author: <di.wang@intel.com>
  33  * Author: <yawei.niu@intel.com>
  34  */
  35 #define DEBUG_SUBSYSTEM S_OST
  36
  37 #include <obd_class.h>
  38 #include <lustre_param.h>
  39 #include <lustre_log.h>
  40
  41 struct lwp_device {
  42         struct lu_device        lpd_dev;
  43         struct obd_device       *lpd_obd;
  44         struct obd_uuid         lpd_cluuid;
  45         struct obd_export       *lpd_exp;
  46         int                     lpd_connects;
  47 };
  48
  49 static inline struct lwp_device *lu2lwp_dev(struct lu_device *d)
  50 {
  51         return container_of0(d, struct lwp_device, lpd_dev);
  52 }
  53
  54 static inline struct lu_device *lwp2lu_dev(struct lwp_device *d)
  55 {
  56         return &d->lpd_dev;
  57 }
  58
  59 static int lwp_name2fsname(char *lwpname, char *fsname)
  60 {
  61         char *ptr;
  62
  63         LASSERT(lwpname != NULL);
  64         LASSERT(fsname != NULL);
  65
  66         sprintf(fsname, "-%s-", LUSTRE_LWP_NAME);
  67
  68         ptr = strstr(lwpname, fsname);
  69         if (ptr == NULL)
  70                 return -EINVAL;
  71
  72         while (*(--ptr) != '-') {
  73                 if (ptr == lwpname)
  74                         return -EINVAL;
  75         }
  76
  77         strncpy(fsname, lwpname, ptr - lwpname);
  78         fsname[ptr - lwpname] = '\0';
  79
  80         return 0;
  81 }
  82
  83 static int lwp_setup(const struct lu_env *env, struct lwp_device *lwp,
  84                      char *nidstring)
  85 {
  86         struct lustre_cfg_bufs  *bufs = NULL;
  87         struct lustre_cfg       *lcfg = NULL;
  88         char                    *lwpname = lwp->lpd_obd->obd_name;
  89         char                    *fsname = NULL;
  90         char                    *server_uuid = NULL;
  91         class_uuid_t             uuid;
  92         struct obd_import       *imp;
  93         int                      rc;
  94         ENTRY;
  95
  96         OBD_ALLOC_PTR(bufs);
  97         if (bufs == NULL)
  98                 RETURN(-ENOMEM);
  99
 100         OBD_ALLOC(fsname, strlen(lwpname));
 101         if (fsname == NULL)
 102                 GOTO(out, rc = -ENOMEM);
 103
 104         rc = lwp_name2fsname(lwpname, fsname);
 105         if (rc) {
 106                 CERROR("%s: failed to get fsname from lwpname. %d\n",
 107                        lwpname, rc);
 108                 GOTO(out, rc);
 109         }
 110
 111         OBD_ALLOC(server_uuid, strlen(fsname) + 15);
 112         if (server_uuid == NULL)
 113                 GOTO(out, rc = -ENOMEM);
 114
 115         sprintf(server_uuid, "%s-MDT0000_UUID", fsname);
 116         lustre_cfg_bufs_reset(bufs, lwpname);
 117         lustre_cfg_bufs_set_string(bufs, 1, server_uuid);
 118         lustre_cfg_bufs_set_string(bufs, 2, nidstring);
 119         lcfg = lustre_cfg_new(LCFG_SETUP, bufs);
 120         if (lcfg == NULL)
 121                 GOTO(out, rc = -ENOMEM);
 122
 123         rc = client_obd_setup(lwp->lpd_obd, lcfg);
 124         if (rc != 0) {
 125                 CERROR("%s: client obd setup error: rc = %d\n",
 126                        lwp->lpd_obd->obd_name, rc);
 127                 GOTO(out, rc);
 128         }
 129
 130         imp = lwp->lpd_obd->u.cli.cl_import;
 131         rc = ptlrpc_init_import(imp);
 132         if (rc)
 133                 GOTO(out, rc);
 134
 135         ll_generate_random_uuid(uuid);
 136         class_uuid_unparse(uuid, &lwp->lpd_cluuid);
 137 out:
 138         if (bufs != NULL)
 139                 OBD_FREE_PTR(bufs);
 140         if (server_uuid != NULL)
 141                 OBD_FREE(server_uuid, strlen(fsname) + 15);
 142         if (fsname != NULL)
 143                 OBD_FREE(fsname, strlen(lwpname));
 144         if (lcfg != NULL)
 145                 lustre_cfg_free(lcfg);
 146         if (rc)
 147                 client_obd_cleanup(lwp->lpd_obd);
 148
 149         RETURN(rc);
 150 }
 151
 152 int lwp_disconnect(struct lwp_device *d)
 153 {
 154         struct obd_import *imp;
 155         int rc = 0;
 156
 157         imp = d->lpd_obd->u.cli.cl_import;
 158
 159         /* Mark import deactivated now, so we don't try to reconnect if any
 160          * of the cleanup RPCs fails (e.g. ldlm cancel, etc).  We don't
 161          * fully deactivate the import, or that would drop all requests. */
 162         LASSERT(imp != NULL);
 163         spin_lock(&imp->imp_lock);
 164         imp->imp_deactive = 1;
 165         spin_unlock(&imp->imp_lock);
 166
 167         ptlrpc_deactivate_import(imp);
 168
 169         /* Some non-replayable imports (MDS's OSCs) are pinged, so just
 170          * delete it regardless.  (It's safe to delete an import that was
 171          * never added.) */
 172         (void)ptlrpc_pinger_del_import(imp);
 173
 174         rc = ptlrpc_disconnect_import(imp, 0);
 175         if (rc && rc != -ETIMEDOUT)
 176                 CERROR("%s: can't disconnect: rc = %d\n",
 177                        d->lpd_obd->obd_name, rc);
 178
 179         ptlrpc_invalidate_import(imp);
 180
 181         RETURN(rc);
 182 }
 183
 184 static int lwp_process_config(const struct lu_env *env,
 185                               struct lu_device *dev, struct lustre_cfg *lcfg)
 186 {
 187         struct lwp_device               *d = lu2lwp_dev(dev);
 188         int                              rc;
 189         ENTRY;
 190
 191         switch (lcfg->lcfg_command) {
 192         case LCFG_PRE_CLEANUP:
 193         case LCFG_CLEANUP:
 194                 rc = lwp_disconnect(d);
 195                 break;
 196         case LCFG_PARAM:
 197                 rc = -ENOSYS;
 198                 break;
 199         default:
 200                 CERROR("%s: unknown command %u\n",
 201                        (char *)lustre_cfg_string(lcfg, 0), lcfg->lcfg_command);
 202                 rc = 0;
 203                 break;
 204         }
 205
 206         RETURN(rc);
 207 }
 208
 209 const struct lu_device_operations lwp_lu_ops = {
 210         .ldo_process_config     = lwp_process_config,
 211 };
 212
 213 static struct lprocfs_seq_vars lprocfs_lwp_obd_vars[] = {
 214         { 0 }
 215 };
 216
 217 int lwp_init0(const struct lu_env *env, struct lwp_device *lwp,
 218               struct lu_device_type *ldt, struct lustre_cfg *cfg)
 219 {
 220         int                        rc;
 221         ENTRY;
 222
 223         lwp->lpd_obd = class_name2obd(lustre_cfg_string(cfg, 0));
 224         if (lwp->lpd_obd == NULL) {
 225                 CERROR("Cannot find obd with name %s\n",
 226                        lustre_cfg_string(cfg, 0));
 227                 RETURN(-ENODEV);
 228         }
 229
 230         lwp->lpd_dev.ld_ops = &lwp_lu_ops;
 231         lwp->lpd_obd->obd_lu_dev = &lwp->lpd_dev;
 232
 233         rc = ptlrpcd_addref();
 234         if (rc) {
 235                 CERROR("%s: ptlrpcd addref error: rc =%d\n",
 236                        lwp->lpd_obd->obd_name, rc);
 237                 RETURN(rc);
 238         }
 239
 240         rc = lwp_setup(env, lwp, lustre_cfg_string(cfg, 1));
 241         if (rc) {
 242                 CERROR("%s: setup lwp failed. %d\n",
 243                        lwp->lpd_obd->obd_name, rc);
 244                 ptlrpcd_decref();
 245                 RETURN(rc);
 246         }
 247
 248         lwp->lpd_obd->obd_vars = lprocfs_lwp_obd_vars;
 249         if (lprocfs_seq_obd_setup(lwp->lpd_obd) == 0)
 250                 ptlrpc_lprocfs_register_obd(lwp->lpd_obd);
 251
 252         RETURN(0);
 253 }
 254
 255 static struct lu_device *lwp_device_free(const struct lu_env *env,
 256                                          struct lu_device *lu)
 257 {
 258         struct lwp_device *m = lu2lwp_dev(lu);
 259         ENTRY;
 260
 261         if (cfs_atomic_read(&lu->ld_ref) && lu->ld_site) {
 262                 LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);
 263                 lu_site_print(env, lu->ld_site, &msgdata, lu_cdebug_printer);
 264         }
 265         lu_device_fini(&m->lpd_dev);
 266         OBD_FREE_PTR(m);
 267         RETURN(NULL);
 268 }
 269
 270 static struct lu_device *lwp_device_alloc(const struct lu_env *env,
 271                                           struct lu_device_type *t,
 272                                           struct lustre_cfg *lcfg)
 273 {
 274         struct lwp_device *lwp;
 275         struct lu_device  *l;
 276
 277         OBD_ALLOC_PTR(lwp);
 278         if (lwp == NULL) {
 279                 l = ERR_PTR(-ENOMEM);
 280         } else {
 281                 int rc;
 282
 283                 l = lwp2lu_dev(lwp);
 284                 lu_device_init(&lwp->lpd_dev, t);
 285                 rc = lwp_init0(env, lwp, t, lcfg);
 286                 if (rc != 0) {
 287                         lwp_device_free(env, l);
 288                         l = ERR_PTR(rc);
 289                 }
 290         }
 291         return l;
 292 }
 293
 294
 295 static struct lu_device *lwp_device_fini(const struct lu_env *env,
 296                                          struct lu_device *d)
 297 {
 298         struct lwp_device *m = lu2lwp_dev(d);
 299         struct obd_import *imp;
 300         int                rc;
 301         ENTRY;
 302
 303         if (m->lpd_exp != NULL)
 304                 class_disconnect(m->lpd_exp);
 305
 306         imp = m->lpd_obd->u.cli.cl_import;
 307
 308         if (imp->imp_rq_pool) {
 309                 ptlrpc_free_rq_pool(imp->imp_rq_pool);
 310                 imp->imp_rq_pool = NULL;
 311         }
 312
 313         LASSERT(m->lpd_obd);
 314         ptlrpc_lprocfs_unregister_obd(m->lpd_obd);
 315         lprocfs_obd_cleanup(m->lpd_obd);
 316
 317         rc = client_obd_cleanup(m->lpd_obd);
 318         LASSERTF(rc == 0, "error %d\n", rc);
 319
 320         ptlrpcd_decref();
 321
 322         RETURN(NULL);
 323 }
 324
 325 static struct lu_device_type_operations lwp_device_type_ops = {
 326         .ldto_device_alloc   = lwp_device_alloc,
 327         .ldto_device_free    = lwp_device_free,
 328         .ldto_device_fini    = lwp_device_fini
 329 };
 330
 331 struct lu_device_type lwp_device_type = {
 332         .ldt_tags     = LU_DEVICE_DT,
 333         .ldt_name     = LUSTRE_LWP_NAME,
 334         .ldt_ops      = &lwp_device_type_ops,
 335         .ldt_ctx_tags = LCT_MD_THREAD
 336 };
 337
 338 static int lwp_obd_connect(const struct lu_env *env, struct obd_export **exp,
 339                            struct obd_device *obd, struct obd_uuid *cluuid,
 340                            struct obd_connect_data *data, void *localdata)
 341 {
 342         struct lwp_device       *lwp = lu2lwp_dev(obd->obd_lu_dev);
 343         struct obd_connect_data *ocd;
 344         struct obd_import       *imp;
 345         struct lustre_handle     conn;
 346         int                      rc;
 347
 348         ENTRY;
 349
 350         CDEBUG(D_CONFIG, "connect #%d\n", lwp->lpd_connects);
 351
 352         rc = class_connect(&conn, obd, cluuid);
 353         if (rc)
 354                 RETURN(rc);
 355
 356         *exp = class_conn2export(&conn);
 357         lwp->lpd_exp = *exp;
 358
 359         /* Why should there ever be more than 1 connect? */
 360         lwp->lpd_connects++;
 361         LASSERT(lwp->lpd_connects == 1);
 362
 363         imp = lwp->lpd_obd->u.cli.cl_import;
 364         imp->imp_dlm_handle = conn;
 365
 366         LASSERT(data != NULL);
 367         ocd = &imp->imp_connect_data;
 368         *ocd = *data;
 369
 370         LASSERT(ocd->ocd_connect_flags & OBD_CONNECT_LIGHTWEIGHT);
 371
 372         ocd->ocd_version = LUSTRE_VERSION_CODE;
 373         imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
 374
 375         rc = ptlrpc_connect_import(imp);
 376         if (rc) {
 377                 CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc);
 378                 GOTO(out, rc);
 379         }
 380
 381         ptlrpc_pinger_add_import(imp);
 382
 383 out:
 384         RETURN(rc);
 385 }
 386
 387 static int lwp_obd_disconnect(struct obd_export *exp)
 388 {
 389         struct obd_device *obd = exp->exp_obd;
 390         struct lwp_device *lwp = lu2lwp_dev(obd->obd_lu_dev);
 391         int                rc;
 392         ENTRY;
 393
 394         /* Only disconnect the underlying layers on the final disconnect. */
 395         LASSERT(lwp->lpd_connects == 1);
 396         lwp->lpd_connects--;
 397
 398         rc = class_disconnect(exp);
 399         if (rc)
 400                 CERROR("%s: class disconnect error: rc = %d\n",
 401                        obd->obd_name, rc);
 402
 403         RETURN(rc);
 404 }
 405
 406 static int lwp_import_event(struct obd_device *obd, struct obd_import *imp,
 407                             enum obd_import_event event)
 408 {
 409         switch (event) {
 410         case IMP_EVENT_DISCON:
 411         case IMP_EVENT_INACTIVE:
 412         case IMP_EVENT_ACTIVE:
 413                 break;
 414         case IMP_EVENT_INVALIDATE:
 415                 if (obd->obd_namespace == NULL)
 416                         break;
 417                 ldlm_namespace_cleanup(obd->obd_namespace, LDLM_FL_LOCAL_ONLY);
 418                 break;
 419         case IMP_EVENT_OCD:
 420                 break;
 421         default:
 422                 CERROR("%s: unsupported import event: %#x\n",
 423                        obd->obd_name, event);
 424         }
 425         return 0;
 426 }
 427
 428 struct obd_ops lwp_obd_device_ops = {
 429         .o_owner        = THIS_MODULE,
 430         .o_add_conn     = client_import_add_conn,
 431         .o_del_conn     = client_import_del_conn,
 432         .o_connect      = lwp_obd_connect,
 433         .o_disconnect   = lwp_obd_disconnect,
 434         .o_import_event = lwp_import_event,
 435 };