lustre/osp/osp_dev.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  *
  30  * Copyright (c) 2011, 2012, Intel, Inc.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * lustre/osp/osp_dev.c
  37  *
  38  * Lustre OST Proxy Device
  39  *
  40  * Author: Alex Zhuravlev <alexey.zhuravlev@intel.com>
  41  * Author: Mikhail Pershin <mike.pershin@intel.com>
  42  */
  43
  44 #ifndef EXPORT_SYMTAB
  45 # define EXPORT_SYMTAB
  46 #endif
  47 #define DEBUG_SUBSYSTEM S_MDS
  48
  49 #include <obd_class.h>
  50 #include <lustre_param.h>
  51
  52 #include "osp_internal.h"
  53
  54 /* Slab for OSP object allocation */
  55 cfs_mem_cache_t *osp_object_kmem;
  56
  57 static struct lu_kmem_descr osp_caches[] = {
  58         {
  59                 .ckd_cache = &osp_object_kmem,
  60                 .ckd_name  = "osp_obj",
  61                 .ckd_size  = sizeof(struct osp_object)
  62         },
  63         {
  64                 .ckd_cache = NULL
  65         }
  66 };
  67
  68 struct lu_object *osp_object_alloc(const struct lu_env *env,
  69                                    const struct lu_object_header *hdr,
  70                                    struct lu_device *d)
  71 {
  72         struct lu_object_header *h;
  73         struct osp_object       *o;
  74         struct lu_object        *l;
  75
  76         LASSERT(hdr == NULL);
  77
  78         OBD_SLAB_ALLOC_PTR_GFP(o, osp_object_kmem, CFS_ALLOC_IO);
  79         if (o != NULL) {
  80                 l = &o->opo_obj.do_lu;
  81                 h = &o->opo_header;
  82
  83                 lu_object_header_init(h);
  84                 dt_object_init(&o->opo_obj, h, d);
  85                 lu_object_add_top(h, l);
  86
  87                 l->lo_ops = &osp_lu_obj_ops;
  88
  89                 return l;
  90         } else {
  91                 return NULL;
  92         }
  93 }
  94
  95 /* Update opd_last_used_id along with checking for gap in objid sequence */
  96 void osp_update_last_id(struct osp_device *d, obd_id objid)
  97 {
  98         /*
  99          * we might have lost precreated objects due to VBR and precreate
 100          * orphans, the gap in objid can be calculated properly only here
 101          */
 102         if (objid > le64_to_cpu(d->opd_last_used_id)) {
 103                 if (objid - le64_to_cpu(d->opd_last_used_id) > 1) {
 104                         d->opd_gap_start = le64_to_cpu(d->opd_last_used_id) + 1;
 105                         d->opd_gap_count = objid - d->opd_gap_start;
 106                         CDEBUG(D_HA, "Gap in objids: %d, start = %llu\n",
 107                                d->opd_gap_count, d->opd_gap_start);
 108                 }
 109                 d->opd_last_used_id = cpu_to_le64(objid);
 110         }
 111 }
 112
 113 static int osp_last_used_init(const struct lu_env *env, struct osp_device *m)
 114 {
 115         struct osp_thread_info  *osi = osp_env_info(env);
 116         struct dt_object_format  dof = { 0 };
 117         struct dt_object        *o;
 118         int                      rc;
 119
 120         ENTRY;
 121
 122         osi->osi_attr.la_valid = LA_MODE;
 123         osi->osi_attr.la_mode = S_IFREG | 0644;
 124         lu_local_obj_fid(&osi->osi_fid, MDD_LOV_OBJ_OID);
 125         dof.dof_type = DFT_REGULAR;
 126         o = dt_find_or_create(env, m->opd_storage, &osi->osi_fid, &dof,
 127                               &osi->osi_attr);
 128         if (IS_ERR(o))
 129                 RETURN(PTR_ERR(o));
 130
 131         rc = dt_attr_get(env, o, &osi->osi_attr, NULL);
 132         if (rc)
 133                 GOTO(out, rc);
 134
 135         /* object will be released in device cleanup path */
 136         m->opd_last_used_file = o;
 137
 138         if (osi->osi_attr.la_size >= sizeof(osi->osi_id) *
 139                                      (m->opd_index + 1)) {
 140                 osp_objid_buf_prep(osi, m, m->opd_index);
 141                 rc = dt_record_read(env, o, &osi->osi_lb, &osi->osi_off);
 142                 if (rc != 0)
 143                         GOTO(out, rc);
 144         } else {
 145                 /* reset value to 0, just to make sure and change file's size */
 146                 struct thandle *th;
 147
 148                 m->opd_last_used_id = 0;
 149                 osp_objid_buf_prep(osi, m, m->opd_index);
 150
 151                 th = dt_trans_create(env, m->opd_storage);
 152                 if (IS_ERR(th))
 153                         GOTO(out, rc = PTR_ERR(th));
 154
 155                 rc = dt_declare_record_write(env, m->opd_last_used_file,
 156                                              osi->osi_lb.lb_len, osi->osi_off,
 157                                              th);
 158                 if (rc) {
 159                         dt_trans_stop(env, m->opd_storage, th);
 160                         GOTO(out, rc);
 161                 }
 162
 163                 rc = dt_trans_start_local(env, m->opd_storage, th);
 164                 if (rc) {
 165                         dt_trans_stop(env, m->opd_storage, th);
 166                         GOTO(out, rc);
 167                 }
 168
 169                 rc = dt_record_write(env, m->opd_last_used_file, &osi->osi_lb,
 170                                      &osi->osi_off, th);
 171                 dt_trans_stop(env, m->opd_storage, th);
 172                 if (rc)
 173                         GOTO(out, rc);
 174         }
 175         RETURN(0);
 176 out:
 177         CERROR("%s: can't initialize lov_objid: %d\n",
 178                m->opd_obd->obd_name, rc);
 179         lu_object_put(env, &o->do_lu);
 180         m->opd_last_used_file = NULL;
 181         return rc;
 182 }
 183
 184 static void osp_last_used_fini(const struct lu_env *env, struct osp_device *d)
 185 {
 186         if (d->opd_last_used_file != NULL) {
 187                 lu_object_put(env, &d->opd_last_used_file->do_lu);
 188                 d->opd_last_used_file = NULL;
 189         }
 190 }
 191
 192 int osp_disconnect(struct osp_device *d)
 193 {
 194         struct obd_import *imp;
 195         int rc = 0;
 196
 197         imp = d->opd_obd->u.cli.cl_import;
 198
 199         /* Mark import deactivated now, so we don't try to reconnect if any
 200          * of the cleanup RPCs fails (e.g. ldlm cancel, etc).  We don't
 201          * fully deactivate the import, or that would drop all requests. */
 202         LASSERT(imp != NULL);
 203         cfs_spin_lock(&imp->imp_lock);
 204         imp->imp_deactive = 1;
 205         cfs_spin_unlock(&imp->imp_lock);
 206
 207         ptlrpc_deactivate_import(imp);
 208
 209         /* Some non-replayable imports (MDS's OSCs) are pinged, so just
 210          * delete it regardless.  (It's safe to delete an import that was
 211          * never added.) */
 212         (void)ptlrpc_pinger_del_import(imp);
 213
 214         rc = ptlrpc_disconnect_import(imp, 0);
 215         if (rc && rc != -ETIMEDOUT)
 216                 CERROR("%s: can't disconnect: rc = %d\n",
 217                        d->opd_obd->obd_name, rc);
 218
 219         ptlrpc_invalidate_import(imp);
 220
 221         RETURN(rc);
 222 }
 223
 224 static int osp_shutdown(const struct lu_env *env, struct osp_device *d)
 225 {
 226         int                      rc = 0;
 227         ENTRY;
 228
 229         if (is_osp_on_ost(d->opd_obd->obd_name)) {
 230                 rc = osp_disconnect(d);
 231                 RETURN(rc);
 232         }
 233
 234         LASSERT(env);
 235         /* release last_used file */
 236         osp_last_used_fini(env, d);
 237
 238         rc = osp_disconnect(d);
 239
 240         /* stop precreate thread */
 241         osp_precreate_fini(d);
 242
 243         /* stop sync thread */
 244         osp_sync_fini(d);
 245
 246         RETURN(rc);
 247 }
 248
 249 static int osp_process_config(const struct lu_env *env,
 250                               struct lu_device *dev, struct lustre_cfg *lcfg)
 251 {
 252         struct osp_device               *d = lu2osp_dev(dev);
 253         struct lprocfs_static_vars       lvars = { 0 };
 254         int                              rc;
 255
 256         ENTRY;
 257
 258         switch (lcfg->lcfg_command) {
 259         case LCFG_CLEANUP:
 260                 if (!is_osp_on_ost(d->opd_obd->obd_name))
 261                         lu_dev_del_linkage(dev->ld_site, dev);
 262                 rc = osp_shutdown(env, d);
 263                 break;
 264         case LCFG_PARAM:
 265                 lprocfs_osp_init_vars(&lvars);
 266
 267                 LASSERT(d->opd_obd);
 268                 rc = class_process_proc_param(PARAM_OSC, lvars.obd_vars,
 269                                               lcfg, d->opd_obd);
 270                 if (rc > 0)
 271                         rc = 0;
 272                 if (rc == -ENOSYS) {
 273                         /* class_process_proc_param() haven't found matching
 274                          * parameter and returned ENOSYS so that layer(s)
 275                          * below could use that. But OSP is the bottom, so
 276                          * just ignore it */
 277                         CERROR("%s: unknown param %s\n",
 278                                (char *)lustre_cfg_string(lcfg, 0),
 279                                (char *)lustre_cfg_string(lcfg, 1));
 280                         rc = 0;
 281                 }
 282                 break;
 283         default:
 284                 CERROR("%s: unknown command %u\n",
 285                        (char *)lustre_cfg_string(lcfg, 0), lcfg->lcfg_command);
 286                 rc = 0;
 287                 break;
 288         }
 289
 290         RETURN(rc);
 291 }
 292
 293 static int osp_recovery_complete(const struct lu_env *env,
 294                                  struct lu_device *dev)
 295 {
 296         struct osp_device       *osp = lu2osp_dev(dev);
 297         int                      rc = 0;
 298
 299         ENTRY;
 300         osp->opd_recovery_completed = 1;
 301         cfs_waitq_signal(&osp->opd_pre_waitq);
 302         RETURN(rc);
 303 }
 304
 305 const struct lu_device_operations osp_lu_ops = {
 306         .ldo_object_alloc       = osp_object_alloc,
 307         .ldo_process_config     = osp_process_config,
 308         .ldo_recovery_complete  = osp_recovery_complete,
 309 };
 310
 311 /**
 312  * provides with statfs from corresponded OST
 313  *
 314  */
 315 static int osp_statfs(const struct lu_env *env, struct dt_device *dev,
 316                       struct obd_statfs *sfs)
 317 {
 318         struct osp_device *d = dt2osp_dev(dev);
 319
 320         ENTRY;
 321
 322         if (unlikely(d->opd_imp_active == 0))
 323                 RETURN(-ENOTCONN);
 324
 325         /* return recently updated data */
 326         *sfs = d->opd_statfs;
 327
 328         /*
 329          * layer above osp (usually lod) can use ffree to estimate
 330          * how many objects are available for immediate creation
 331          */
 332         cfs_spin_lock(&d->opd_pre_lock);
 333         sfs->os_fprecreated = d->opd_pre_last_created - d->opd_pre_used_id;
 334         sfs->os_fprecreated -= d->opd_pre_reserved;
 335         cfs_spin_unlock(&d->opd_pre_lock);
 336
 337         LASSERT(sfs->os_fprecreated <= OST_MAX_PRECREATE);
 338
 339         CDEBUG(D_OTHER, "%s: "LPU64" blocks, "LPU64" free, "LPU64" avail, "
 340                LPU64" files, "LPU64" free files\n", d->opd_obd->obd_name,
 341                sfs->os_blocks, sfs->os_bfree, sfs->os_bavail,
 342                sfs->os_files, sfs->os_ffree);
 343         RETURN(0);
 344 }
 345
 346 static int osp_sync(const struct lu_env *env, struct dt_device *dev)
 347 {
 348         ENTRY;
 349
 350         /*
 351          * XXX: wake up sync thread, command it to start flushing asap?
 352          */
 353
 354         RETURN(0);
 355 }
 356
 357 const struct dt_device_operations osp_dt_ops = {
 358         .dt_statfs      = osp_statfs,
 359         .dt_sync        = osp_sync,
 360 };
 361
 362 static int osp_connect_to_osd(const struct lu_env *env, struct osp_device *m,
 363                               const char *nextdev)
 364 {
 365         struct obd_connect_data *data = NULL;
 366         struct obd_device       *obd;
 367         int                      rc;
 368
 369         ENTRY;
 370
 371         LASSERT(m->opd_storage_exp == NULL);
 372
 373         OBD_ALLOC_PTR(data);
 374         if (data == NULL)
 375                 RETURN(-ENOMEM);
 376
 377         obd = class_name2obd(nextdev);
 378         if (obd == NULL) {
 379                 CERROR("%s: can't locate next device: %s\n",
 380                        m->opd_obd->obd_name, nextdev);
 381                 GOTO(out, rc = -ENOTCONN);
 382         }
 383
 384         rc = obd_connect(env, &m->opd_storage_exp, obd, &obd->obd_uuid, data,
 385                          NULL);
 386         if (rc) {
 387                 CERROR("%s: cannot connect to next dev %s: rc = %d\n",
 388                        m->opd_obd->obd_name, nextdev, rc);
 389                 GOTO(out, rc);
 390         }
 391
 392         m->opd_dt_dev.dd_lu_dev.ld_site =
 393                 m->opd_storage_exp->exp_obd->obd_lu_dev->ld_site;
 394         LASSERT(m->opd_dt_dev.dd_lu_dev.ld_site);
 395         m->opd_storage = lu2dt_dev(m->opd_storage_exp->exp_obd->obd_lu_dev);
 396
 397 out:
 398         OBD_FREE_PTR(data);
 399         RETURN(rc);
 400 }
 401
 402 static int osp_init0(const struct lu_env *env, struct osp_device *m,
 403                      struct lu_device_type *ldt, struct lustre_cfg *cfg)
 404 {
 405         struct lprocfs_static_vars       lvars = { 0 };
 406         struct proc_dir_entry           *osc_proc_dir;
 407         struct obd_import               *imp;
 408         class_uuid_t                     uuid;
 409         char                            *src, *ost, *mdt, *osdname = NULL;
 410         int                              rc, idx;
 411
 412         ENTRY;
 413
 414         m->opd_obd = class_name2obd(lustre_cfg_string(cfg, 0));
 415         if (m->opd_obd == NULL) {
 416                 CERROR("Cannot find obd with name %s\n",
 417                        lustre_cfg_string(cfg, 0));
 418                 RETURN(-ENODEV);
 419         }
 420
 421         /* There is no record in the MDT configuration for the local disk
 422          * device, so we have to extract this from elsewhere in the profile.
 423          * The only information we get at setup is from the OSC records:
 424          * setup 0:{fsname}-OSTxxxx-osc[-MDTxxxx] 1:lustre-OST0000_UUID 2:NID
 425          * Note that 1.8 generated configs are missing the -MDTxxxx part.
 426          * We need to reconstruct the name of the underlying OSD from this:
 427          * {fsname}-{svname}-osd, for example "lustre-MDT0000-osd".  We
 428          * also need to determine the OST index from this - will be used
 429          * to calculate the offset in shared lov_objids file later */
 430
 431         src = lustre_cfg_string(cfg, 0);
 432         if (src == NULL)
 433                 RETURN(-EINVAL);
 434
 435         ost = strstr(src, "-OST");
 436         if (ost == NULL)
 437                 RETURN(-EINVAL);
 438
 439         idx = simple_strtol(ost + 4, &mdt, 16);
 440         if (mdt[0] != '-' || idx > INT_MAX || idx < 0) {
 441                 CERROR("%s: invalid OST index in '%s'\n",
 442                        m->opd_obd->obd_name, src);
 443                 GOTO(out_fini, rc = -EINVAL);
 444         }
 445         m->opd_index = idx;
 446
 447         idx = ost - src;
 448         /* check the fsname length, and after this everything else will fit */
 449         if (idx > MTI_NAME_MAXLEN) {
 450                 CERROR("%s: fsname too long in '%s'\n",
 451                        m->opd_obd->obd_name, src);
 452                 GOTO(out_fini, rc = -EINVAL);
 453         }
 454
 455         OBD_ALLOC(osdname, MAX_OBD_NAME);
 456         if (osdname == NULL)
 457                 GOTO(out_fini, rc = -ENOMEM);
 458
 459         memcpy(osdname, src, idx); /* copy just the fsname part */
 460         osdname[idx] = '\0';
 461
 462         mdt = strstr(mdt, "-MDT");
 463         if (mdt == NULL) /* 1.8 configs don't have "-MDT0000" at the end */
 464                 strcat(osdname, "-MDT0000");
 465         else
 466                 strcat(osdname, mdt);
 467         strcat(osdname, "-osd");
 468         CDEBUG(D_HA, "%s: connect to %s (%s)\n",
 469                m->opd_obd->obd_name, osdname, src);
 470
 471         m->opd_dt_dev.dd_lu_dev.ld_ops = &osp_lu_ops;
 472         m->opd_dt_dev.dd_ops = &osp_dt_ops;
 473         m->opd_obd->obd_lu_dev = &m->opd_dt_dev.dd_lu_dev;
 474
 475         rc = osp_connect_to_osd(env, m, osdname);
 476         if (rc)
 477                 GOTO(out_fini, rc);
 478
 479         rc = ptlrpcd_addref();
 480         if (rc)
 481                 GOTO(out_disconnect, rc);
 482
 483         rc = client_obd_setup(m->opd_obd, cfg);
 484         if (rc) {
 485                 CERROR("%s: can't setup obd: %d\n", m->opd_obd->obd_name, rc);
 486                 GOTO(out_ref, rc);
 487         }
 488
 489         lprocfs_osp_init_vars(&lvars);
 490         if (lprocfs_obd_setup(m->opd_obd, lvars.obd_vars) == 0)
 491                 ptlrpc_lprocfs_register_obd(m->opd_obd);
 492
 493         /* for compatibility we link old procfs's OSC entries to osp ones */
 494         osc_proc_dir = lprocfs_srch(proc_lustre_root, "osc");
 495         if (osc_proc_dir) {
 496                 cfs_proc_dir_entry_t    *symlink = NULL;
 497                 char                    *name;
 498
 499                 OBD_ALLOC(name, strlen(m->opd_obd->obd_name) + 1);
 500                 if (name == NULL)
 501                         GOTO(out, rc = -ENOMEM);
 502
 503                 strcpy(name, m->opd_obd->obd_name);
 504                 if (strstr(name, "osc"))
 505                         symlink = lprocfs_add_symlink(name, osc_proc_dir,
 506                                                       "../osp/%s",
 507                                                       m->opd_obd->obd_name);
 508                 OBD_FREE(name, strlen(m->opd_obd->obd_name) + 1);
 509                 m->opd_symlink = symlink;
 510         }
 511
 512         /*
 513          * Initialize last id from the storage - will be used in orphan cleanup
 514          */
 515         rc = osp_last_used_init(env, m);
 516         if (rc)
 517                 GOTO(out_proc, rc);
 518
 519         /*
 520          * Initialize precreation thread, it handles new connections as well
 521          */
 522         rc = osp_init_precreate(m);
 523         if (rc)
 524                 GOTO(out_last_used, rc);
 525
 526         /*
 527          * Initialize synhronization mechanism taking care of propogating
 528          * changes to OST in near transactional manner
 529          */
 530         rc = osp_sync_init(env, m);
 531         if (rc)
 532                 GOTO(out_precreat, rc);
 533
 534         /*
 535          * Initiate connect to OST
 536          */
 537         ll_generate_random_uuid(uuid);
 538         class_uuid_unparse(uuid, &m->opd_cluuid);
 539
 540         imp = m->opd_obd->u.cli.cl_import;
 541
 542         rc = ptlrpc_init_import(imp);
 543         if (rc)
 544                 GOTO(out, rc);
 545         if (osdname)
 546                 OBD_FREE(osdname, MAX_OBD_NAME);
 547         RETURN(0);
 548
 549 out:
 550         /* stop sync thread */
 551         osp_sync_fini(m);
 552 out_precreat:
 553         /* stop precreate thread */
 554         osp_precreate_fini(m);
 555 out_last_used:
 556         osp_last_used_fini(env, m);
 557 out_proc:
 558         ptlrpc_lprocfs_unregister_obd(m->opd_obd);
 559         lprocfs_obd_cleanup(m->opd_obd);
 560         class_destroy_import(m->opd_obd->u.cli.cl_import);
 561         client_obd_cleanup(m->opd_obd);
 562 out_ref:
 563         ptlrpcd_decref();
 564 out_disconnect:
 565         obd_disconnect(m->opd_storage_exp);
 566 out_fini:
 567         if (osdname)
 568                 OBD_FREE(osdname, MAX_OBD_NAME);
 569         RETURN(rc);
 570 }
 571
 572 static struct lu_device *osp_device_free(const struct lu_env *env,
 573                                          struct lu_device *lu)
 574 {
 575         struct osp_device *m = lu2osp_dev(lu);
 576
 577         ENTRY;
 578
 579         if (cfs_atomic_read(&lu->ld_ref) && lu->ld_site) {
 580                 LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);
 581                 lu_site_print(env, lu->ld_site, &msgdata, lu_cdebug_printer);
 582         }
 583         dt_device_fini(&m->opd_dt_dev);
 584         OBD_FREE_PTR(m);
 585         RETURN(NULL);
 586 }
 587
 588 static struct lu_device *osp_device_alloc(const struct lu_env *env,
 589                                           struct lu_device_type *t,
 590                                           struct lustre_cfg *lcfg)
 591 {
 592         struct osp_device *m;
 593         struct lu_device  *l;
 594
 595         OBD_ALLOC_PTR(m);
 596         if (m == NULL) {
 597                 l = ERR_PTR(-ENOMEM);
 598         } else {
 599                 int rc;
 600
 601                 l = osp2lu_dev(m);
 602                 dt_device_init(&m->opd_dt_dev, t);
 603                 if (is_osp_on_ost(lustre_cfg_string(lcfg, 0)))
 604                         rc = osp_init_for_ost(env, m, t, lcfg);
 605                 else
 606                         rc = osp_init0(env, m, t, lcfg);
 607                 if (rc != 0) {
 608                         osp_device_free(env, l);
 609                         l = ERR_PTR(rc);
 610                 }
 611         }
 612         return l;
 613 }
 614
 615 static struct lu_device *osp_device_fini(const struct lu_env *env,
 616                                          struct lu_device *d)
 617 {
 618         struct osp_device *m = lu2osp_dev(d);
 619         struct obd_import *imp;
 620         int                rc;
 621
 622         ENTRY;
 623
 624         if (m->opd_storage_exp)
 625                 obd_disconnect(m->opd_storage_exp);
 626
 627         if (is_osp_on_ost(m->opd_obd->obd_name))
 628                 osp_fini_for_ost(m);
 629
 630         imp = m->opd_obd->u.cli.cl_import;
 631
 632         if (imp->imp_rq_pool) {
 633                 ptlrpc_free_rq_pool(imp->imp_rq_pool);
 634                 imp->imp_rq_pool = NULL;
 635         }
 636
 637         obd_cleanup_client_import(m->opd_obd);
 638
 639         if (m->opd_symlink)
 640                 lprocfs_remove(&m->opd_symlink);
 641
 642         LASSERT(m->opd_obd);
 643         ptlrpc_lprocfs_unregister_obd(m->opd_obd);
 644         lprocfs_obd_cleanup(m->opd_obd);
 645
 646         rc = client_obd_cleanup(m->opd_obd);
 647         LASSERTF(rc == 0, "error %d\n", rc);
 648
 649         ptlrpcd_decref();
 650
 651         RETURN(NULL);
 652 }
 653
 654 static int osp_reconnect(const struct lu_env *env,
 655                          struct obd_export *exp, struct obd_device *obd,
 656                          struct obd_uuid *cluuid,
 657                          struct obd_connect_data *data,
 658                          void *localdata)
 659 {
 660         return 0;
 661 }
 662
 663 /*
 664  * we use exports to track all LOD users
 665  */
 666 static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp,
 667                            struct obd_device *obd, struct obd_uuid *cluuid,
 668                            struct obd_connect_data *data, void *localdata)
 669 {
 670         struct osp_device       *osp = lu2osp_dev(obd->obd_lu_dev);
 671         struct obd_connect_data *ocd;
 672         struct obd_import       *imp;
 673         struct lustre_handle     conn;
 674         int                      rc;
 675
 676         ENTRY;
 677
 678         CDEBUG(D_CONFIG, "connect #%d\n", osp->opd_connects);
 679
 680         rc = class_connect(&conn, obd, cluuid);
 681         if (rc)
 682                 RETURN(rc);
 683
 684         *exp = class_conn2export(&conn);
 685         if (is_osp_on_ost(obd->obd_name))
 686                 osp->opd_exp = *exp;
 687
 688         /* Why should there ever be more than 1 connect? */
 689         osp->opd_connects++;
 690         LASSERT(osp->opd_connects == 1);
 691
 692         imp = osp->opd_obd->u.cli.cl_import;
 693         imp->imp_dlm_handle = conn;
 694
 695         ocd = &imp->imp_connect_data;
 696         ocd->ocd_connect_flags = OBD_CONNECT_AT |
 697                                  OBD_CONNECT_FULL20 |
 698                                  OBD_CONNECT_INDEX |
 699 #ifdef HAVE_LRU_RESIZE_SUPPORT
 700                                  OBD_CONNECT_LRU_RESIZE |
 701 #endif
 702                                  OBD_CONNECT_MDS |
 703                                  OBD_CONNECT_OSS_CAPA |
 704                                  OBD_CONNECT_REQPORTAL |
 705                                  OBD_CONNECT_SKIP_ORPHAN |
 706                                  OBD_CONNECT_VERSION |
 707                                  OBD_CONNECT_FID;
 708
 709         if (is_osp_on_ost(osp->opd_obd->obd_name))
 710                 ocd->ocd_connect_flags |= OBD_CONNECT_LIGHTWEIGHT;
 711
 712         ocd->ocd_version = LUSTRE_VERSION_CODE;
 713         LASSERT(data->ocd_connect_flags & OBD_CONNECT_INDEX);
 714         ocd->ocd_index = data->ocd_index;
 715         imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
 716
 717         rc = ptlrpc_connect_import(imp);
 718         if (rc) {
 719                 CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc);
 720                 GOTO(out, rc);
 721         }
 722
 723         ptlrpc_pinger_add_import(imp);
 724
 725 out:
 726         RETURN(rc);
 727 }
 728
 729 /*
 730  * once last export (we don't count self-export) disappeared
 731  * osp can be released
 732  */
 733 static int osp_obd_disconnect(struct obd_export *exp)
 734 {
 735         struct obd_device *obd = exp->exp_obd;
 736         struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev);
 737         int                rc;
 738         ENTRY;
 739
 740         /* Only disconnect the underlying layers on the final disconnect. */
 741         LASSERT(osp->opd_connects == 1);
 742         osp->opd_connects--;
 743
 744         rc = class_disconnect(exp);
 745         if (rc) {
 746                 CERROR("%s: class disconnect error: rc = %d\n",
 747                        obd->obd_name, rc);
 748                 RETURN(rc);
 749         }
 750
 751         /* destroy the device */
 752         if (!is_osp_on_ost(obd->obd_name))
 753                 class_manual_cleanup(obd);
 754
 755         RETURN(rc);
 756 }
 757
 758 /*
 759  * lprocfs helpers still use OBD API, let's keep obd_statfs() support
 760  */
 761 static int osp_obd_statfs(const struct lu_env *env, struct obd_export *exp,
 762                           struct obd_statfs *osfs, __u64 max_age, __u32 flags)
 763 {
 764         struct obd_statfs       *msfs;
 765         struct ptlrpc_request   *req;
 766         struct obd_import       *imp = NULL;
 767         int                      rc;
 768
 769         ENTRY;
 770
 771         /* Since the request might also come from lprocfs, so we need
 772          * sync this with client_disconnect_export Bug15684 */
 773         cfs_down_read(&exp->exp_obd->u.cli.cl_sem);
 774         if (exp->exp_obd->u.cli.cl_import)
 775                 imp = class_import_get(exp->exp_obd->u.cli.cl_import);
 776         cfs_up_read(&exp->exp_obd->u.cli.cl_sem);
 777         if (!imp)
 778                 RETURN(-ENODEV);
 779
 780         /* We could possibly pass max_age in the request (as an absolute
 781          * timestamp or a "seconds.usec ago") so the target can avoid doing
 782          * extra calls into the filesystem if that isn't necessary (e.g.
 783          * during mount that would help a bit).  Having relative timestamps
 784          * is not so great if request processing is slow, while absolute
 785          * timestamps are not ideal because they need time synchronization. */
 786         req = ptlrpc_request_alloc(imp, &RQF_OST_STATFS);
 787
 788         class_import_put(imp);
 789
 790         if (req == NULL)
 791                 RETURN(-ENOMEM);
 792
 793         rc = ptlrpc_request_pack(req, LUSTRE_OST_VERSION, OST_STATFS);
 794         if (rc) {
 795                 ptlrpc_request_free(req);
 796                 RETURN(rc);
 797         }
 798         ptlrpc_request_set_replen(req);
 799         req->rq_request_portal = OST_CREATE_PORTAL;
 800         ptlrpc_at_set_req_timeout(req);
 801
 802         if (flags & OBD_STATFS_NODELAY) {
 803                 /* procfs requests not want stat in wait for avoid deadlock */
 804                 req->rq_no_resend = 1;
 805                 req->rq_no_delay = 1;
 806         }
 807
 808         rc = ptlrpc_queue_wait(req);
 809         if (rc)
 810                 GOTO(out, rc);
 811
 812         msfs = req_capsule_server_get(&req->rq_pill, &RMF_OBD_STATFS);
 813         if (msfs == NULL)
 814                 GOTO(out, rc = -EPROTO);
 815
 816         *osfs = *msfs;
 817
 818         EXIT;
 819 out:
 820         ptlrpc_req_finished(req);
 821         return rc;
 822 }
 823
 824 static int osp_import_event(struct obd_device *obd, struct obd_import *imp,
 825                             enum obd_import_event event)
 826 {
 827         struct osp_device *d = lu2osp_dev(obd->obd_lu_dev);
 828
 829         switch (event) {
 830         case IMP_EVENT_DISCON:
 831                 d->opd_got_disconnected = 1;
 832                 d->opd_imp_connected = 0;
 833                 if (is_osp_on_ost(d->opd_obd->obd_name))
 834                         break;
 835                 osp_pre_update_status(d, -ENODEV);
 836                 cfs_waitq_signal(&d->opd_pre_waitq);
 837                 CDEBUG(D_HA, "got disconnected\n");
 838                 break;
 839         case IMP_EVENT_INACTIVE:
 840                 d->opd_imp_active = 0;
 841                 if (is_osp_on_ost(d->opd_obd->obd_name))
 842                         break;
 843                 osp_pre_update_status(d, -ENODEV);
 844                 cfs_waitq_signal(&d->opd_pre_waitq);
 845                 CDEBUG(D_HA, "got inactive\n");
 846                 break;
 847         case IMP_EVENT_ACTIVE:
 848                 d->opd_imp_active = 1;
 849                 if (d->opd_got_disconnected)
 850                         d->opd_new_connection = 1;
 851                 d->opd_imp_connected = 1;
 852                 d->opd_imp_seen_connected = 1;
 853                 if (is_osp_on_ost(d->opd_obd->obd_name))
 854                         break;
 855                 cfs_waitq_signal(&d->opd_pre_waitq);
 856                 __osp_sync_check_for_work(d);
 857                 CDEBUG(D_HA, "got connected\n");
 858                 break;
 859         case IMP_EVENT_INVALIDATE:
 860                 if (obd->obd_namespace == NULL)
 861                         break;
 862                 ldlm_namespace_cleanup(obd->obd_namespace, LDLM_FL_LOCAL_ONLY);
 863                 break;
 864         case IMP_EVENT_OCD:
 865                 break;
 866         default:
 867                 CERROR("%s: unsupported import event: %#x\n",
 868                        obd->obd_name, event);
 869         }
 870         return 0;
 871 }
 872
 873 static int osp_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 874                          void *karg, void *uarg)
 875 {
 876         struct obd_device       *obd = exp->exp_obd;
 877         struct osp_device       *d;
 878         struct obd_ioctl_data   *data = karg;
 879         int                      rc = 0;
 880
 881         ENTRY;
 882
 883         LASSERT(obd->obd_lu_dev);
 884         d = lu2osp_dev(obd->obd_lu_dev);
 885         LASSERT(d->opd_dt_dev.dd_ops == &osp_dt_ops);
 886
 887         if (!cfs_try_module_get(THIS_MODULE)) {
 888                 CERROR("%s: can't get module. Is it alive?", obd->obd_name);
 889                 return -EINVAL;
 890         }
 891
 892         switch (cmd) {
 893         case OBD_IOC_CLIENT_RECOVER:
 894                 rc = ptlrpc_recover_import(obd->u.cli.cl_import,
 895                                            data->ioc_inlbuf1, 0);
 896                 if (rc > 0)
 897                         rc = 0;
 898                 break;
 899         case IOC_OSC_SET_ACTIVE:
 900                 rc = ptlrpc_set_import_active(obd->u.cli.cl_import,
 901                                               data->ioc_offset);
 902                 break;
 903         case OBD_IOC_PING_TARGET:
 904                 rc = ptlrpc_obd_ping(obd);
 905                 break;
 906         default:
 907                 CERROR("%s: unrecognized ioctl %#x by %s\n", obd->obd_name,
 908                        cmd, cfs_curproc_comm());
 909                 rc = -ENOTTY;
 910         }
 911         cfs_module_put(THIS_MODULE);
 912         return rc;
 913 }
 914
 915 static int osp_obd_health_check(const struct lu_env *env,
 916                                 struct obd_device *obd)
 917 {
 918         struct osp_device *d = lu2osp_dev(obd->obd_lu_dev);
 919
 920         ENTRY;
 921
 922         /*
 923          * 1.8/2.0 behaviour is that OST being connected once at least
 924          * is considired "healthy". and one "healty" OST is enough to
 925          * allow lustre clients to connect to MDS
 926          */
 927         LASSERT(d);
 928         RETURN(!d->opd_imp_seen_connected);
 929 }
 930
 931 /* context key constructor/destructor: mdt_key_init, mdt_key_fini */
 932 LU_KEY_INIT_FINI(osp, struct osp_thread_info);
 933 static void osp_key_exit(const struct lu_context *ctx,
 934                          struct lu_context_key *key, void *data)
 935 {
 936         struct osp_thread_info *info = data;
 937
 938         info->osi_attr.la_valid = 0;
 939 }
 940
 941 struct lu_context_key osp_thread_key = {
 942         .lct_tags = LCT_MD_THREAD,
 943         .lct_init = osp_key_init,
 944         .lct_fini = osp_key_fini,
 945         .lct_exit = osp_key_exit
 946 };
 947
 948 /* context key constructor/destructor: mdt_txn_key_init, mdt_txn_key_fini */
 949 LU_KEY_INIT_FINI(osp_txn, struct osp_txn_info);
 950
 951 struct lu_context_key osp_txn_key = {
 952         .lct_tags = LCT_OSP_THREAD,
 953         .lct_init = osp_txn_key_init,
 954         .lct_fini = osp_txn_key_fini
 955 };
 956 LU_TYPE_INIT_FINI(osp, &osp_thread_key, &osp_txn_key);
 957
 958 static struct lu_device_type_operations osp_device_type_ops = {
 959         .ldto_init           = osp_type_init,
 960         .ldto_fini           = osp_type_fini,
 961
 962         .ldto_start          = osp_type_start,
 963         .ldto_stop           = osp_type_stop,
 964
 965         .ldto_device_alloc   = osp_device_alloc,
 966         .ldto_device_free    = osp_device_free,
 967
 968         .ldto_device_fini    = osp_device_fini
 969 };
 970
 971 static struct lu_device_type osp_device_type = {
 972         .ldt_tags     = LU_DEVICE_DT,
 973         .ldt_name     = LUSTRE_OSP_NAME,
 974         .ldt_ops      = &osp_device_type_ops,
 975         .ldt_ctx_tags = LCT_MD_THREAD
 976 };
 977
 978 static struct obd_ops osp_obd_device_ops = {
 979         .o_owner        = THIS_MODULE,
 980         .o_add_conn     = client_import_add_conn,
 981         .o_del_conn     = client_import_del_conn,
 982         .o_reconnect    = osp_reconnect,
 983         .o_connect      = osp_obd_connect,
 984         .o_disconnect   = osp_obd_disconnect,
 985         .o_health_check = osp_obd_health_check,
 986         .o_import_event = osp_import_event,
 987         .o_iocontrol    = osp_iocontrol,
 988         .o_statfs       = osp_obd_statfs,
 989 };
 990
 991 static int __init osp_mod_init(void)
 992 {
 993         struct lprocfs_static_vars       lvars;
 994         cfs_proc_dir_entry_t            *osc_proc_dir;
 995         int                              rc;
 996
 997         rc = lu_kmem_init(osp_caches);
 998         if (rc)
 999                 return rc;
1000
1001         lprocfs_osp_init_vars(&lvars);
1002
1003         rc = class_register_type(&osp_obd_device_ops, NULL, lvars.module_vars,
1004                                  LUSTRE_OSP_NAME, &osp_device_type);
1005
1006         /* create "osc" entry in procfs for compatibility purposes */
1007         if (rc != 0) {
1008                 lu_kmem_fini(osp_caches);
1009                 return rc;
1010         }
1011
1012         osc_proc_dir = lprocfs_srch(proc_lustre_root, "osc");
1013         if (osc_proc_dir == NULL) {
1014                 osc_proc_dir = lprocfs_register("osc", proc_lustre_root, NULL,
1015                                                 NULL);
1016                 if (IS_ERR(osc_proc_dir))
1017                         CERROR("osp: can't create compat entry \"osc\": %d\n",
1018                                (int) PTR_ERR(osc_proc_dir));
1019         }
1020         return rc;
1021 }
1022
1023 static void __exit osp_mod_exit(void)
1024 {
1025         lprocfs_try_remove_proc_entry("osc", proc_lustre_root);
1026
1027         class_unregister_type(LUSTRE_OSP_NAME);
1028         lu_kmem_fini(osp_caches);
1029 }
1030
1031 MODULE_AUTHOR("Intel, Inc. <http://www.intel.com/>");
1032 MODULE_DESCRIPTION("Lustre OST Proxy Device ("LUSTRE_OSP_NAME")");
1033 MODULE_LICENSE("GPL");
1034
1035 cfs_module(osp, LUSTRE_VERSION_STRING, osp_mod_init, osp_mod_exit);
1036