lustre/osp/osp_dev.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  *
  30  * Copyright (c) 2012, Intel Corporation.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * lustre/osp/osp_dev.c
  37  *
  38  * Lustre OST Proxy Device
  39  *
  40  * Author: Alex Zhuravlev <alexey.zhuravlev@intel.com>
  41  * Author: Mikhail Pershin <mike.pershin@intel.com>
  42  */
  43
  44 #ifndef EXPORT_SYMTAB
  45 # define EXPORT_SYMTAB
  46 #endif
  47 #define DEBUG_SUBSYSTEM S_MDS
  48
  49 #include <obd_class.h>
  50 #include <lustre_param.h>
  51 #include <lustre_log.h>
  52
  53 #include "osp_internal.h"
  54
  55 /* Slab for OSP object allocation */
  56 cfs_mem_cache_t *osp_object_kmem;
  57
  58 static struct lu_kmem_descr osp_caches[] = {
  59         {
  60                 .ckd_cache = &osp_object_kmem,
  61                 .ckd_name  = "osp_obj",
  62                 .ckd_size  = sizeof(struct osp_object)
  63         },
  64         {
  65                 .ckd_cache = NULL
  66         }
  67 };
  68
  69 struct lu_object *osp_object_alloc(const struct lu_env *env,
  70                                    const struct lu_object_header *hdr,
  71                                    struct lu_device *d)
  72 {
  73         struct lu_object_header *h;
  74         struct osp_object       *o;
  75         struct lu_object        *l;
  76
  77         LASSERT(hdr == NULL);
  78
  79         OBD_SLAB_ALLOC_PTR_GFP(o, osp_object_kmem, CFS_ALLOC_IO);
  80         if (o != NULL) {
  81                 l = &o->opo_obj.do_lu;
  82                 h = &o->opo_header;
  83
  84                 lu_object_header_init(h);
  85                 dt_object_init(&o->opo_obj, h, d);
  86                 lu_object_add_top(h, l);
  87
  88                 l->lo_ops = &osp_lu_obj_ops;
  89
  90                 return l;
  91         } else {
  92                 return NULL;
  93         }
  94 }
  95
  96 /* Update opd_last_used_id along with checking for gap in objid sequence */
  97 void osp_update_last_id(struct osp_device *d, obd_id objid)
  98 {
  99         /*
 100          * we might have lost precreated objects due to VBR and precreate
 101          * orphans, the gap in objid can be calculated properly only here
 102          */
 103         if (objid > le64_to_cpu(d->opd_last_used_id)) {
 104                 if (objid - le64_to_cpu(d->opd_last_used_id) > 1) {
 105                         d->opd_gap_start = le64_to_cpu(d->opd_last_used_id) + 1;
 106                         d->opd_gap_count = objid - d->opd_gap_start;
 107                         CDEBUG(D_HA, "Gap in objids: %d, start = %llu\n",
 108                                d->opd_gap_count, d->opd_gap_start);
 109                 }
 110                 d->opd_last_used_id = cpu_to_le64(objid);
 111         }
 112 }
 113
 114 static int osp_last_used_init(const struct lu_env *env, struct osp_device *m)
 115 {
 116         struct osp_thread_info  *osi = osp_env_info(env);
 117         struct dt_object_format  dof = { 0 };
 118         struct dt_object        *o;
 119         int                      rc;
 120
 121         ENTRY;
 122
 123         osi->osi_attr.la_valid = LA_MODE;
 124         osi->osi_attr.la_mode = S_IFREG | 0644;
 125         lu_local_obj_fid(&osi->osi_fid, MDD_LOV_OBJ_OID);
 126         dof.dof_type = DFT_REGULAR;
 127         o = dt_find_or_create(env, m->opd_storage, &osi->osi_fid, &dof,
 128                               &osi->osi_attr);
 129         if (IS_ERR(o))
 130                 RETURN(PTR_ERR(o));
 131
 132         rc = dt_attr_get(env, o, &osi->osi_attr, NULL);
 133         if (rc)
 134                 GOTO(out, rc);
 135
 136         /* object will be released in device cleanup path */
 137         m->opd_last_used_file = o;
 138
 139         if (osi->osi_attr.la_size >= sizeof(osi->osi_id) *
 140                                      (m->opd_index + 1)) {
 141                 osp_objid_buf_prep(osi, m, m->opd_index);
 142                 rc = dt_record_read(env, o, &osi->osi_lb, &osi->osi_off);
 143                 if (rc != 0)
 144                         GOTO(out, rc);
 145         } else {
 146                 /* reset value to 0, just to make sure and change file's size */
 147                 struct thandle *th;
 148
 149                 m->opd_last_used_id = 0;
 150                 osp_objid_buf_prep(osi, m, m->opd_index);
 151
 152                 th = dt_trans_create(env, m->opd_storage);
 153                 if (IS_ERR(th))
 154                         GOTO(out, rc = PTR_ERR(th));
 155
 156                 rc = dt_declare_record_write(env, m->opd_last_used_file,
 157                                              osi->osi_lb.lb_len, osi->osi_off,
 158                                              th);
 159                 if (rc) {
 160                         dt_trans_stop(env, m->opd_storage, th);
 161                         GOTO(out, rc);
 162                 }
 163
 164                 rc = dt_trans_start_local(env, m->opd_storage, th);
 165                 if (rc) {
 166                         dt_trans_stop(env, m->opd_storage, th);
 167                         GOTO(out, rc);
 168                 }
 169
 170                 rc = dt_record_write(env, m->opd_last_used_file, &osi->osi_lb,
 171                                      &osi->osi_off, th);
 172                 dt_trans_stop(env, m->opd_storage, th);
 173                 if (rc)
 174                         GOTO(out, rc);
 175         }
 176         CDEBUG(D_HA, "%s: Read last used ID: "LPU64"\n", m->opd_obd->obd_name,
 177                le64_to_cpu(m->opd_last_used_id));
 178         RETURN(0);
 179 out:
 180         CERROR("%s: can't initialize lov_objid: %d\n",
 181                m->opd_obd->obd_name, rc);
 182         lu_object_put(env, &o->do_lu);
 183         m->opd_last_used_file = NULL;
 184         return rc;
 185 }
 186
 187 static void osp_last_used_fini(const struct lu_env *env, struct osp_device *d)
 188 {
 189         if (d->opd_last_used_file != NULL) {
 190                 lu_object_put(env, &d->opd_last_used_file->do_lu);
 191                 d->opd_last_used_file = NULL;
 192         }
 193 }
 194
 195 int osp_disconnect(struct osp_device *d)
 196 {
 197         struct obd_import *imp;
 198         int rc = 0;
 199
 200         imp = d->opd_obd->u.cli.cl_import;
 201
 202         /* Mark import deactivated now, so we don't try to reconnect if any
 203          * of the cleanup RPCs fails (e.g. ldlm cancel, etc).  We don't
 204          * fully deactivate the import, or that would drop all requests. */
 205         LASSERT(imp != NULL);
 206         spin_lock(&imp->imp_lock);
 207         imp->imp_deactive = 1;
 208         spin_unlock(&imp->imp_lock);
 209
 210         ptlrpc_deactivate_import(imp);
 211
 212         /* Some non-replayable imports (MDS's OSCs) are pinged, so just
 213          * delete it regardless.  (It's safe to delete an import that was
 214          * never added.) */
 215         (void)ptlrpc_pinger_del_import(imp);
 216
 217         rc = ptlrpc_disconnect_import(imp, 0);
 218         if (rc == -ETIMEDOUT || rc == -ENOTCONN || rc == -ESHUTDOWN)
 219                 rc = 0;
 220         if (rc)
 221                 CERROR("%s: can't disconnect: rc = %d\n",
 222                        d->opd_obd->obd_name, rc);
 223
 224         ptlrpc_invalidate_import(imp);
 225
 226         RETURN(rc);
 227 }
 228
 229 static int osp_shutdown(const struct lu_env *env, struct osp_device *d)
 230 {
 231         int                      rc = 0;
 232         ENTRY;
 233
 234         if (is_osp_on_ost(d->opd_obd->obd_name)) {
 235                 rc = osp_disconnect(d);
 236                 RETURN(rc);
 237         }
 238
 239         LASSERT(env);
 240         /* release last_used file */
 241         osp_last_used_fini(env, d);
 242
 243         rc = osp_disconnect(d);
 244
 245         /* stop precreate thread */
 246         osp_precreate_fini(d);
 247
 248         /* stop sync thread */
 249         osp_sync_fini(d);
 250
 251         obd_fid_fini(d->opd_obd);
 252
 253         RETURN(rc);
 254 }
 255
 256 static int osp_process_config(const struct lu_env *env,
 257                               struct lu_device *dev, struct lustre_cfg *lcfg)
 258 {
 259         struct osp_device               *d = lu2osp_dev(dev);
 260         struct lprocfs_static_vars       lvars = { 0 };
 261         int                              rc;
 262
 263         ENTRY;
 264
 265         switch (lcfg->lcfg_command) {
 266         case LCFG_CLEANUP:
 267                 if (!is_osp_on_ost(d->opd_obd->obd_name))
 268                         lu_dev_del_linkage(dev->ld_site, dev);
 269                 rc = osp_shutdown(env, d);
 270                 break;
 271         case LCFG_PARAM:
 272                 lprocfs_osp_init_vars(&lvars);
 273
 274                 LASSERT(d->opd_obd);
 275                 rc = class_process_proc_param(PARAM_OSC, lvars.obd_vars,
 276                                               lcfg, d->opd_obd);
 277                 if (rc > 0)
 278                         rc = 0;
 279                 if (rc == -ENOSYS) {
 280                         /* class_process_proc_param() haven't found matching
 281                          * parameter and returned ENOSYS so that layer(s)
 282                          * below could use that. But OSP is the bottom, so
 283                          * just ignore it */
 284                         CERROR("%s: unknown param %s\n",
 285                                (char *)lustre_cfg_string(lcfg, 0),
 286                                (char *)lustre_cfg_string(lcfg, 1));
 287                         rc = 0;
 288                 }
 289                 break;
 290         default:
 291                 CERROR("%s: unknown command %u\n",
 292                        (char *)lustre_cfg_string(lcfg, 0), lcfg->lcfg_command);
 293                 rc = 0;
 294                 break;
 295         }
 296
 297         RETURN(rc);
 298 }
 299
 300 static int osp_recovery_complete(const struct lu_env *env,
 301                                  struct lu_device *dev)
 302 {
 303         struct osp_device       *osp = lu2osp_dev(dev);
 304         int                      rc = 0;
 305
 306         ENTRY;
 307         osp->opd_recovery_completed = 1;
 308         cfs_waitq_signal(&osp->opd_pre_waitq);
 309         RETURN(rc);
 310 }
 311
 312 const struct lu_device_operations osp_lu_ops = {
 313         .ldo_object_alloc       = osp_object_alloc,
 314         .ldo_process_config     = osp_process_config,
 315         .ldo_recovery_complete  = osp_recovery_complete,
 316 };
 317
 318 /**
 319  * provides with statfs from corresponded OST
 320  *
 321  */
 322 static int osp_statfs(const struct lu_env *env, struct dt_device *dev,
 323                       struct obd_statfs *sfs)
 324 {
 325         struct osp_device *d = dt2osp_dev(dev);
 326
 327         ENTRY;
 328
 329         if (unlikely(d->opd_imp_active == 0))
 330                 RETURN(-ENOTCONN);
 331
 332         /* return recently updated data */
 333         *sfs = d->opd_statfs;
 334
 335         /*
 336          * layer above osp (usually lod) can use ffree to estimate
 337          * how many objects are available for immediate creation
 338          */
 339         spin_lock(&d->opd_pre_lock);
 340         sfs->os_fprecreated = d->opd_pre_last_created - d->opd_pre_used_id;
 341         sfs->os_fprecreated -= d->opd_pre_reserved;
 342         spin_unlock(&d->opd_pre_lock);
 343
 344         LASSERT(sfs->os_fprecreated <= OST_MAX_PRECREATE * 2);
 345
 346         CDEBUG(D_OTHER, "%s: "LPU64" blocks, "LPU64" free, "LPU64" avail, "
 347                LPU64" files, "LPU64" free files\n", d->opd_obd->obd_name,
 348                sfs->os_blocks, sfs->os_bfree, sfs->os_bavail,
 349                sfs->os_files, sfs->os_ffree);
 350         RETURN(0);
 351 }
 352
 353 static int osp_sync(const struct lu_env *env, struct dt_device *dev)
 354 {
 355         ENTRY;
 356
 357         /*
 358          * XXX: wake up sync thread, command it to start flushing asap?
 359          */
 360
 361         RETURN(0);
 362 }
 363
 364 const struct dt_device_operations osp_dt_ops = {
 365         .dt_statfs      = osp_statfs,
 366         .dt_sync        = osp_sync,
 367 };
 368
 369 static int osp_connect_to_osd(const struct lu_env *env, struct osp_device *m,
 370                               const char *nextdev)
 371 {
 372         struct obd_connect_data *data = NULL;
 373         struct obd_device       *obd;
 374         int                      rc;
 375
 376         ENTRY;
 377
 378         LASSERT(m->opd_storage_exp == NULL);
 379
 380         OBD_ALLOC_PTR(data);
 381         if (data == NULL)
 382                 RETURN(-ENOMEM);
 383
 384         obd = class_name2obd(nextdev);
 385         if (obd == NULL) {
 386                 CERROR("%s: can't locate next device: %s\n",
 387                        m->opd_obd->obd_name, nextdev);
 388                 GOTO(out, rc = -ENOTCONN);
 389         }
 390
 391         rc = obd_connect(env, &m->opd_storage_exp, obd, &obd->obd_uuid, data,
 392                          NULL);
 393         if (rc) {
 394                 CERROR("%s: cannot connect to next dev %s: rc = %d\n",
 395                        m->opd_obd->obd_name, nextdev, rc);
 396                 GOTO(out, rc);
 397         }
 398
 399         m->opd_dt_dev.dd_lu_dev.ld_site =
 400                 m->opd_storage_exp->exp_obd->obd_lu_dev->ld_site;
 401         LASSERT(m->opd_dt_dev.dd_lu_dev.ld_site);
 402         m->opd_storage = lu2dt_dev(m->opd_storage_exp->exp_obd->obd_lu_dev);
 403
 404 out:
 405         OBD_FREE_PTR(data);
 406         RETURN(rc);
 407 }
 408
 409 static int osp_init0(const struct lu_env *env, struct osp_device *m,
 410                      struct lu_device_type *ldt, struct lustre_cfg *cfg)
 411 {
 412         struct obd_device               *obd;
 413         struct obd_import               *imp;
 414         class_uuid_t                     uuid;
 415         char                            *src, *ost, *mdt, *osdname = NULL;
 416         int                              rc, idx;
 417
 418         ENTRY;
 419
 420         obd = class_name2obd(lustre_cfg_string(cfg, 0));
 421         if (obd == NULL) {
 422                 CERROR("Cannot find obd with name %s\n",
 423                        lustre_cfg_string(cfg, 0));
 424                 RETURN(-ENODEV);
 425         }
 426         m->opd_obd = obd;
 427
 428         /* There is no record in the MDT configuration for the local disk
 429          * device, so we have to extract this from elsewhere in the profile.
 430          * The only information we get at setup is from the OSC records:
 431          * setup 0:{fsname}-OSTxxxx-osc[-MDTxxxx] 1:lustre-OST0000_UUID 2:NID
 432          * Note that 1.8 generated configs are missing the -MDTxxxx part.
 433          * We need to reconstruct the name of the underlying OSD from this:
 434          * {fsname}-{svname}-osd, for example "lustre-MDT0000-osd".  We
 435          * also need to determine the OST index from this - will be used
 436          * to calculate the offset in shared lov_objids file later */
 437
 438         src = lustre_cfg_string(cfg, 0);
 439         if (src == NULL)
 440                 RETURN(-EINVAL);
 441
 442         ost = strstr(src, "-OST");
 443         if (ost == NULL)
 444                 RETURN(-EINVAL);
 445
 446         idx = simple_strtol(ost + 4, &mdt, 16);
 447         if (mdt[0] != '-' || idx > INT_MAX || idx < 0) {
 448                 CERROR("%s: invalid OST index in '%s'\n", obd->obd_name, src);
 449                 GOTO(out_fini, rc = -EINVAL);
 450         }
 451         m->opd_index = idx;
 452
 453         idx = ost - src;
 454         /* check the fsname length, and after this everything else will fit */
 455         if (idx > MTI_NAME_MAXLEN) {
 456                 CERROR("%s: fsname too long in '%s'\n", obd->obd_name, src);
 457                 GOTO(out_fini, rc = -EINVAL);
 458         }
 459
 460         OBD_ALLOC(osdname, MAX_OBD_NAME);
 461         if (osdname == NULL)
 462                 GOTO(out_fini, rc = -ENOMEM);
 463
 464         memcpy(osdname, src, idx); /* copy just the fsname part */
 465         osdname[idx] = '\0';
 466
 467         mdt = strstr(mdt, "-MDT");
 468         if (mdt == NULL) /* 1.8 configs don't have "-MDT0000" at the end */
 469                 strcat(osdname, "-MDT0000");
 470         else
 471                 strcat(osdname, mdt);
 472         strcat(osdname, "-osd");
 473         CDEBUG(D_HA, "%s: connect to %s (%s)\n", obd->obd_name, osdname, src);
 474
 475         m->opd_dt_dev.dd_lu_dev.ld_ops = &osp_lu_ops;
 476         m->opd_dt_dev.dd_ops = &osp_dt_ops;
 477         obd->obd_lu_dev = &m->opd_dt_dev.dd_lu_dev;
 478
 479         rc = osp_connect_to_osd(env, m, osdname);
 480         if (rc)
 481                 GOTO(out_fini, rc);
 482
 483         rc = ptlrpcd_addref();
 484         if (rc)
 485                 GOTO(out_disconnect, rc);
 486
 487         rc = client_obd_setup(obd, cfg);
 488         if (rc) {
 489                 CERROR("%s: can't setup obd: %d\n", m->opd_obd->obd_name, rc);
 490                 GOTO(out_ref, rc);
 491         }
 492
 493         osp_lprocfs_init(m);
 494
 495         /*
 496          * Initialize last id from the storage - will be used in orphan cleanup
 497          */
 498         rc = osp_last_used_init(env, m);
 499         if (rc)
 500                 GOTO(out_proc, rc);
 501
 502         /*
 503          * Initialize precreation thread, it handles new connections as well
 504          */
 505         rc = osp_init_precreate(m);
 506         if (rc)
 507                 GOTO(out_last_used, rc);
 508
 509         /*
 510          * Initialize synhronization mechanism taking care of propogating
 511          * changes to OST in near transactional manner
 512          */
 513         rc = osp_sync_init(env, m);
 514         if (rc)
 515                 GOTO(out_precreat, rc);
 516
 517         rc = obd_fid_init(m->opd_obd, NULL, LUSTRE_SEQ_DATA);
 518         if (rc) {
 519                 CERROR("%s: fid init error: rc = %d\n",
 520                        m->opd_obd->obd_name, rc);
 521                 GOTO(out, rc);
 522         }
 523
 524         /*
 525          * Initiate connect to OST
 526          */
 527         ll_generate_random_uuid(uuid);
 528         class_uuid_unparse(uuid, &m->opd_cluuid);
 529
 530         imp = obd->u.cli.cl_import;
 531
 532         rc = ptlrpc_init_import(imp);
 533         if (rc)
 534                 GOTO(out, rc);
 535         if (osdname)
 536                 OBD_FREE(osdname, MAX_OBD_NAME);
 537         RETURN(0);
 538
 539 out:
 540         /* stop sync thread */
 541         osp_sync_fini(m);
 542 out_precreat:
 543         /* stop precreate thread */
 544         osp_precreate_fini(m);
 545 out_last_used:
 546         osp_last_used_fini(env, m);
 547 out_proc:
 548         ptlrpc_lprocfs_unregister_obd(obd);
 549         lprocfs_obd_cleanup(obd);
 550         class_destroy_import(obd->u.cli.cl_import);
 551         client_obd_cleanup(obd);
 552 out_ref:
 553         ptlrpcd_decref();
 554 out_disconnect:
 555         obd_disconnect(m->opd_storage_exp);
 556 out_fini:
 557         if (osdname)
 558                 OBD_FREE(osdname, MAX_OBD_NAME);
 559         RETURN(rc);
 560 }
 561
 562 static struct lu_device *osp_device_free(const struct lu_env *env,
 563                                          struct lu_device *lu)
 564 {
 565         struct osp_device *m = lu2osp_dev(lu);
 566
 567         ENTRY;
 568
 569         if (cfs_atomic_read(&lu->ld_ref) && lu->ld_site) {
 570                 LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);
 571                 lu_site_print(env, lu->ld_site, &msgdata, lu_cdebug_printer);
 572         }
 573         dt_device_fini(&m->opd_dt_dev);
 574         OBD_FREE_PTR(m);
 575         RETURN(NULL);
 576 }
 577
 578 static struct lu_device *osp_device_alloc(const struct lu_env *env,
 579                                           struct lu_device_type *t,
 580                                           struct lustre_cfg *lcfg)
 581 {
 582         struct osp_device *m;
 583         struct lu_device  *l;
 584
 585         OBD_ALLOC_PTR(m);
 586         if (m == NULL) {
 587                 l = ERR_PTR(-ENOMEM);
 588         } else {
 589                 int rc;
 590
 591                 l = osp2lu_dev(m);
 592                 dt_device_init(&m->opd_dt_dev, t);
 593                 if (is_osp_on_ost(lustre_cfg_string(lcfg, 0)))
 594                         rc = osp_init_for_ost(env, m, t, lcfg);
 595                 else
 596                         rc = osp_init0(env, m, t, lcfg);
 597                 if (rc != 0) {
 598                         osp_device_free(env, l);
 599                         l = ERR_PTR(rc);
 600                 }
 601         }
 602         return l;
 603 }
 604
 605 static struct lu_device *osp_device_fini(const struct lu_env *env,
 606                                          struct lu_device *d)
 607 {
 608         struct osp_device *m = lu2osp_dev(d);
 609         struct obd_import *imp;
 610         int                rc;
 611
 612         ENTRY;
 613
 614         if (m->opd_storage_exp)
 615                 obd_disconnect(m->opd_storage_exp);
 616
 617         if (is_osp_on_ost(m->opd_obd->obd_name))
 618                 osp_fini_for_ost(m);
 619
 620         imp = m->opd_obd->u.cli.cl_import;
 621
 622         if (imp->imp_rq_pool) {
 623                 ptlrpc_free_rq_pool(imp->imp_rq_pool);
 624                 imp->imp_rq_pool = NULL;
 625         }
 626
 627         obd_cleanup_client_import(m->opd_obd);
 628
 629         if (m->opd_symlink)
 630                 lprocfs_remove(&m->opd_symlink);
 631
 632         LASSERT(m->opd_obd);
 633         ptlrpc_lprocfs_unregister_obd(m->opd_obd);
 634         lprocfs_obd_cleanup(m->opd_obd);
 635
 636         rc = client_obd_cleanup(m->opd_obd);
 637         LASSERTF(rc == 0, "error %d\n", rc);
 638
 639         ptlrpcd_decref();
 640
 641         RETURN(NULL);
 642 }
 643
 644 static int osp_reconnect(const struct lu_env *env,
 645                          struct obd_export *exp, struct obd_device *obd,
 646                          struct obd_uuid *cluuid,
 647                          struct obd_connect_data *data,
 648                          void *localdata)
 649 {
 650         return 0;
 651 }
 652
 653 /*
 654  * we use exports to track all LOD users
 655  */
 656 static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp,
 657                            struct obd_device *obd, struct obd_uuid *cluuid,
 658                            struct obd_connect_data *data, void *localdata)
 659 {
 660         struct osp_device       *osp = lu2osp_dev(obd->obd_lu_dev);
 661         struct obd_connect_data *ocd;
 662         struct obd_import       *imp;
 663         struct lustre_handle     conn;
 664         int                      rc;
 665
 666         ENTRY;
 667
 668         CDEBUG(D_CONFIG, "connect #%d\n", osp->opd_connects);
 669
 670         rc = class_connect(&conn, obd, cluuid);
 671         if (rc)
 672                 RETURN(rc);
 673
 674         *exp = class_conn2export(&conn);
 675         osp->opd_exp = *exp;
 676
 677         /* Why should there ever be more than 1 connect? */
 678         osp->opd_connects++;
 679         LASSERT(osp->opd_connects == 1);
 680
 681         imp = osp->opd_obd->u.cli.cl_import;
 682         imp->imp_dlm_handle = conn;
 683
 684         ocd = &imp->imp_connect_data;
 685         ocd->ocd_connect_flags = OBD_CONNECT_AT |
 686                                  OBD_CONNECT_FULL20 |
 687                                  OBD_CONNECT_INDEX |
 688 #ifdef HAVE_LRU_RESIZE_SUPPORT
 689                                  OBD_CONNECT_LRU_RESIZE |
 690 #endif
 691                                  OBD_CONNECT_MDS |
 692                                  OBD_CONNECT_OSS_CAPA |
 693                                  OBD_CONNECT_REQPORTAL |
 694                                  OBD_CONNECT_SKIP_ORPHAN |
 695                                  OBD_CONNECT_VERSION |
 696                                  OBD_CONNECT_FID |
 697                                  OBD_CONNECT_LVB_TYPE;
 698
 699         if (is_osp_on_ost(osp->opd_obd->obd_name))
 700                 ocd->ocd_connect_flags |= OBD_CONNECT_LIGHTWEIGHT;
 701
 702         ocd->ocd_version = LUSTRE_VERSION_CODE;
 703         LASSERT(data->ocd_connect_flags & OBD_CONNECT_INDEX);
 704         ocd->ocd_index = data->ocd_index;
 705         imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
 706
 707         rc = ptlrpc_connect_import(imp);
 708         if (rc) {
 709                 CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc);
 710                 GOTO(out, rc);
 711         }
 712
 713         ptlrpc_pinger_add_import(imp);
 714
 715 out:
 716         RETURN(rc);
 717 }
 718
 719 /*
 720  * once last export (we don't count self-export) disappeared
 721  * osp can be released
 722  */
 723 static int osp_obd_disconnect(struct obd_export *exp)
 724 {
 725         struct obd_device *obd = exp->exp_obd;
 726         struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev);
 727         int                rc;
 728         ENTRY;
 729
 730         /* Only disconnect the underlying layers on the final disconnect. */
 731         LASSERT(osp->opd_connects == 1);
 732         osp->opd_connects--;
 733
 734         rc = class_disconnect(exp);
 735         if (rc) {
 736                 CERROR("%s: class disconnect error: rc = %d\n",
 737                        obd->obd_name, rc);
 738                 RETURN(rc);
 739         }
 740
 741         /* destroy the device */
 742         if (!is_osp_on_ost(obd->obd_name))
 743                 class_manual_cleanup(obd);
 744
 745         RETURN(rc);
 746 }
 747
 748 /*
 749  * lprocfs helpers still use OBD API, let's keep obd_statfs() support
 750  */
 751 static int osp_obd_statfs(const struct lu_env *env, struct obd_export *exp,
 752                           struct obd_statfs *osfs, __u64 max_age, __u32 flags)
 753 {
 754         struct obd_statfs       *msfs;
 755         struct ptlrpc_request   *req;
 756         struct obd_import       *imp = NULL;
 757         int                      rc;
 758
 759         ENTRY;
 760
 761         /* Since the request might also come from lprocfs, so we need
 762          * sync this with client_disconnect_export Bug15684 */
 763         down_read(&exp->exp_obd->u.cli.cl_sem);
 764         if (exp->exp_obd->u.cli.cl_import)
 765                 imp = class_import_get(exp->exp_obd->u.cli.cl_import);
 766         up_read(&exp->exp_obd->u.cli.cl_sem);
 767         if (!imp)
 768                 RETURN(-ENODEV);
 769
 770         /* We could possibly pass max_age in the request (as an absolute
 771          * timestamp or a "seconds.usec ago") so the target can avoid doing
 772          * extra calls into the filesystem if that isn't necessary (e.g.
 773          * during mount that would help a bit).  Having relative timestamps
 774          * is not so great if request processing is slow, while absolute
 775          * timestamps are not ideal because they need time synchronization. */
 776         req = ptlrpc_request_alloc(imp, &RQF_OST_STATFS);
 777
 778         class_import_put(imp);
 779
 780         if (req == NULL)
 781                 RETURN(-ENOMEM);
 782
 783         rc = ptlrpc_request_pack(req, LUSTRE_OST_VERSION, OST_STATFS);
 784         if (rc) {
 785                 ptlrpc_request_free(req);
 786                 RETURN(rc);
 787         }
 788         ptlrpc_request_set_replen(req);
 789         req->rq_request_portal = OST_CREATE_PORTAL;
 790         ptlrpc_at_set_req_timeout(req);
 791
 792         if (flags & OBD_STATFS_NODELAY) {
 793                 /* procfs requests not want stat in wait for avoid deadlock */
 794                 req->rq_no_resend = 1;
 795                 req->rq_no_delay = 1;
 796         }
 797
 798         rc = ptlrpc_queue_wait(req);
 799         if (rc)
 800                 GOTO(out, rc);
 801
 802         msfs = req_capsule_server_get(&req->rq_pill, &RMF_OBD_STATFS);
 803         if (msfs == NULL)
 804                 GOTO(out, rc = -EPROTO);
 805
 806         *osfs = *msfs;
 807
 808         EXIT;
 809 out:
 810         ptlrpc_req_finished(req);
 811         return rc;
 812 }
 813
 814 static int osp_import_event(struct obd_device *obd, struct obd_import *imp,
 815                             enum obd_import_event event)
 816 {
 817         struct osp_device *d = lu2osp_dev(obd->obd_lu_dev);
 818
 819         switch (event) {
 820         case IMP_EVENT_DISCON:
 821                 d->opd_got_disconnected = 1;
 822                 d->opd_imp_connected = 0;
 823                 if (is_osp_on_ost(d->opd_obd->obd_name))
 824                         break;
 825                 osp_pre_update_status(d, -ENODEV);
 826                 cfs_waitq_signal(&d->opd_pre_waitq);
 827                 CDEBUG(D_HA, "got disconnected\n");
 828                 break;
 829         case IMP_EVENT_INACTIVE:
 830                 d->opd_imp_active = 0;
 831                 if (is_osp_on_ost(d->opd_obd->obd_name))
 832                         break;
 833                 osp_pre_update_status(d, -ENODEV);
 834                 cfs_waitq_signal(&d->opd_pre_waitq);
 835                 CDEBUG(D_HA, "got inactive\n");
 836                 break;
 837         case IMP_EVENT_ACTIVE:
 838                 d->opd_imp_active = 1;
 839                 if (d->opd_got_disconnected)
 840                         d->opd_new_connection = 1;
 841                 d->opd_imp_connected = 1;
 842                 d->opd_imp_seen_connected = 1;
 843                 if (is_osp_on_ost(d->opd_obd->obd_name))
 844                         break;
 845                 if (d->opd_obd->u.cli.cl_seq->lcs_exp == NULL)
 846                         d->opd_obd->u.cli.cl_seq->lcs_exp =
 847                                         class_export_get(d->opd_exp);
 848                 cfs_waitq_signal(&d->opd_pre_waitq);
 849                 __osp_sync_check_for_work(d);
 850                 CDEBUG(D_HA, "got connected\n");
 851                 break;
 852         case IMP_EVENT_INVALIDATE:
 853                 if (obd->obd_namespace == NULL)
 854                         break;
 855                 ldlm_namespace_cleanup(obd->obd_namespace, LDLM_FL_LOCAL_ONLY);
 856                 break;
 857         case IMP_EVENT_OCD:
 858         case IMP_EVENT_DEACTIVATE:
 859         case IMP_EVENT_ACTIVATE:
 860                 break;
 861         default:
 862                 CERROR("%s: unsupported import event: %#x\n",
 863                        obd->obd_name, event);
 864         }
 865         return 0;
 866 }
 867
 868 static int osp_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 869                          void *karg, void *uarg)
 870 {
 871         struct obd_device       *obd = exp->exp_obd;
 872         struct osp_device       *d;
 873         struct obd_ioctl_data   *data = karg;
 874         int                      rc = 0;
 875
 876         ENTRY;
 877
 878         LASSERT(obd->obd_lu_dev);
 879         d = lu2osp_dev(obd->obd_lu_dev);
 880         LASSERT(d->opd_dt_dev.dd_ops == &osp_dt_ops);
 881
 882         if (!cfs_try_module_get(THIS_MODULE)) {
 883                 CERROR("%s: can't get module. Is it alive?", obd->obd_name);
 884                 return -EINVAL;
 885         }
 886
 887         switch (cmd) {
 888         case OBD_IOC_CLIENT_RECOVER:
 889                 rc = ptlrpc_recover_import(obd->u.cli.cl_import,
 890                                            data->ioc_inlbuf1, 0);
 891                 if (rc > 0)
 892                         rc = 0;
 893                 break;
 894         case IOC_OSC_SET_ACTIVE:
 895                 rc = ptlrpc_set_import_active(obd->u.cli.cl_import,
 896                                               data->ioc_offset);
 897                 break;
 898         case OBD_IOC_PING_TARGET:
 899                 rc = ptlrpc_obd_ping(obd);
 900                 break;
 901         default:
 902                 CERROR("%s: unrecognized ioctl %#x by %s\n", obd->obd_name,
 903                        cmd, cfs_curproc_comm());
 904                 rc = -ENOTTY;
 905         }
 906         cfs_module_put(THIS_MODULE);
 907         return rc;
 908 }
 909
 910 static int osp_obd_health_check(const struct lu_env *env,
 911                                 struct obd_device *obd)
 912 {
 913         struct osp_device *d = lu2osp_dev(obd->obd_lu_dev);
 914
 915         ENTRY;
 916
 917         /*
 918          * 1.8/2.0 behaviour is that OST being connected once at least
 919          * is considired "healthy". and one "healty" OST is enough to
 920          * allow lustre clients to connect to MDS
 921          */
 922         LASSERT(d);
 923         RETURN(!d->opd_imp_seen_connected);
 924 }
 925
 926 /* context key constructor/destructor: mdt_key_init, mdt_key_fini */
 927 LU_KEY_INIT_FINI(osp, struct osp_thread_info);
 928 static void osp_key_exit(const struct lu_context *ctx,
 929                          struct lu_context_key *key, void *data)
 930 {
 931         struct osp_thread_info *info = data;
 932
 933         info->osi_attr.la_valid = 0;
 934 }
 935
 936 struct lu_context_key osp_thread_key = {
 937         .lct_tags = LCT_MD_THREAD,
 938         .lct_init = osp_key_init,
 939         .lct_fini = osp_key_fini,
 940         .lct_exit = osp_key_exit
 941 };
 942
 943 /* context key constructor/destructor: mdt_txn_key_init, mdt_txn_key_fini */
 944 LU_KEY_INIT_FINI(osp_txn, struct osp_txn_info);
 945
 946 struct lu_context_key osp_txn_key = {
 947         .lct_tags = LCT_OSP_THREAD,
 948         .lct_init = osp_txn_key_init,
 949         .lct_fini = osp_txn_key_fini
 950 };
 951 LU_TYPE_INIT_FINI(osp, &osp_thread_key, &osp_txn_key);
 952
 953 static struct lu_device_type_operations osp_device_type_ops = {
 954         .ldto_init           = osp_type_init,
 955         .ldto_fini           = osp_type_fini,
 956
 957         .ldto_start          = osp_type_start,
 958         .ldto_stop           = osp_type_stop,
 959
 960         .ldto_device_alloc   = osp_device_alloc,
 961         .ldto_device_free    = osp_device_free,
 962
 963         .ldto_device_fini    = osp_device_fini
 964 };
 965
 966 static struct lu_device_type osp_device_type = {
 967         .ldt_tags     = LU_DEVICE_DT,
 968         .ldt_name     = LUSTRE_OSP_NAME,
 969         .ldt_ops      = &osp_device_type_ops,
 970         .ldt_ctx_tags = LCT_MD_THREAD
 971 };
 972
 973 static struct obd_ops osp_obd_device_ops = {
 974         .o_owner        = THIS_MODULE,
 975         .o_add_conn     = client_import_add_conn,
 976         .o_del_conn     = client_import_del_conn,
 977         .o_reconnect    = osp_reconnect,
 978         .o_connect      = osp_obd_connect,
 979         .o_disconnect   = osp_obd_disconnect,
 980         .o_health_check = osp_obd_health_check,
 981         .o_import_event = osp_import_event,
 982         .o_iocontrol    = osp_iocontrol,
 983         .o_statfs       = osp_obd_statfs,
 984         .o_fid_init     = client_fid_init,
 985         .o_fid_fini     = client_fid_fini,
 986 };
 987
 988 struct llog_operations osp_mds_ost_orig_logops;
 989
 990 static int __init osp_mod_init(void)
 991 {
 992         struct lprocfs_static_vars       lvars;
 993         cfs_proc_dir_entry_t            *osc_proc_dir;
 994         int                              rc;
 995
 996         rc = lu_kmem_init(osp_caches);
 997         if (rc)
 998                 return rc;
 999
1000         lprocfs_osp_init_vars(&lvars);
1001
1002         rc = class_register_type(&osp_obd_device_ops, NULL, lvars.module_vars,
1003                                  LUSTRE_OSP_NAME, &osp_device_type);
1004
1005         /* create "osc" entry in procfs for compatibility purposes */
1006         if (rc != 0) {
1007                 lu_kmem_fini(osp_caches);
1008                 return rc;
1009         }
1010
1011         /* Note: add_rec/delcare_add_rec will be only used by catalogs */
1012         osp_mds_ost_orig_logops = llog_osd_ops;
1013         osp_mds_ost_orig_logops.lop_add = llog_cat_add_rec;
1014         osp_mds_ost_orig_logops.lop_declare_add = llog_cat_declare_add_rec;
1015
1016         osc_proc_dir = lprocfs_srch(proc_lustre_root, "osc");
1017         if (osc_proc_dir == NULL) {
1018                 osc_proc_dir = lprocfs_register("osc", proc_lustre_root, NULL,
1019                                                 NULL);
1020                 if (IS_ERR(osc_proc_dir))
1021                         CERROR("osp: can't create compat entry \"osc\": %d\n",
1022                                (int) PTR_ERR(osc_proc_dir));
1023         }
1024         return rc;
1025 }
1026
1027 static void __exit osp_mod_exit(void)
1028 {
1029         lprocfs_try_remove_proc_entry("osc", proc_lustre_root);
1030
1031         class_unregister_type(LUSTRE_OSP_NAME);
1032         lu_kmem_fini(osp_caches);
1033 }
1034
1035 MODULE_AUTHOR("Intel, Inc. <http://www.intel.com/>");
1036 MODULE_DESCRIPTION("Lustre OST Proxy Device ("LUSTRE_OSP_NAME")");
1037 MODULE_LICENSE("GPL");
1038
1039 cfs_module(osp, LUSTRE_VERSION_STRING, osp_mod_init, osp_mod_exit);