lustre/osp/osp_dev.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  *
  30  * Copyright (c) 2012, Intel Corporation.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * lustre/osp/osp_dev.c
  37  *
  38  * Lustre OST Proxy Device
  39  *
  40  * Author: Alex Zhuravlev <alexey.zhuravlev@intel.com>
  41  * Author: Mikhail Pershin <mike.pershin@intel.com>
  42  */
  43
  44 #ifndef EXPORT_SYMTAB
  45 # define EXPORT_SYMTAB
  46 #endif
  47 #define DEBUG_SUBSYSTEM S_MDS
  48
  49 #include <obd_class.h>
  50 #include <lustre_param.h>
  51 #include <lustre_log.h>
  52 #include <lustre_mdc.h>
  53
  54 #include "osp_internal.h"
  55
  56 /* Slab for OSP object allocation */
  57 cfs_mem_cache_t *osp_object_kmem;
  58
  59 static struct lu_kmem_descr osp_caches[] = {
  60         {
  61                 .ckd_cache = &osp_object_kmem,
  62                 .ckd_name  = "osp_obj",
  63                 .ckd_size  = sizeof(struct osp_object)
  64         },
  65         {
  66                 .ckd_cache = NULL
  67         }
  68 };
  69
  70 struct lu_object *osp_object_alloc(const struct lu_env *env,
  71                                    const struct lu_object_header *hdr,
  72                                    struct lu_device *d)
  73 {
  74         struct lu_object_header *h;
  75         struct osp_object       *o;
  76         struct lu_object        *l;
  77
  78         LASSERT(hdr == NULL);
  79
  80         OBD_SLAB_ALLOC_PTR_GFP(o, osp_object_kmem, CFS_ALLOC_IO);
  81         if (o != NULL) {
  82                 l = &o->opo_obj.do_lu;
  83                 h = &o->opo_header;
  84
  85                 lu_object_header_init(h);
  86                 dt_object_init(&o->opo_obj, h, d);
  87                 lu_object_add_top(h, l);
  88
  89                 l->lo_ops = &osp_lu_obj_ops;
  90
  91                 return l;
  92         } else {
  93                 return NULL;
  94         }
  95 }
  96
  97 /* Update opd_last_used_id along with checking for gap in objid sequence */
  98 void osp_update_last_id(struct osp_device *d, obd_id objid)
  99 {
 100         /*
 101          * we might have lost precreated objects due to VBR and precreate
 102          * orphans, the gap in objid can be calculated properly only here
 103          */
 104         if (objid > le64_to_cpu(d->opd_last_used_id)) {
 105                 if (objid - le64_to_cpu(d->opd_last_used_id) > 1) {
 106                         d->opd_gap_start = le64_to_cpu(d->opd_last_used_id) + 1;
 107                         d->opd_gap_count = objid - d->opd_gap_start;
 108                         CDEBUG(D_HA, "Gap in objids: %d, start = %llu\n",
 109                                d->opd_gap_count, d->opd_gap_start);
 110                 }
 111                 d->opd_last_used_id = cpu_to_le64(objid);
 112         }
 113 }
 114
 115 static int osp_last_used_init(const struct lu_env *env, struct osp_device *m)
 116 {
 117         struct osp_thread_info  *osi = osp_env_info(env);
 118         struct dt_object_format  dof = { 0 };
 119         struct dt_object        *o;
 120         int                      rc;
 121
 122         ENTRY;
 123
 124         osi->osi_attr.la_valid = LA_MODE;
 125         osi->osi_attr.la_mode = S_IFREG | 0644;
 126         lu_local_obj_fid(&osi->osi_fid, MDD_LOV_OBJ_OID);
 127         dof.dof_type = DFT_REGULAR;
 128         o = dt_find_or_create(env, m->opd_storage, &osi->osi_fid, &dof,
 129                               &osi->osi_attr);
 130         if (IS_ERR(o))
 131                 RETURN(PTR_ERR(o));
 132
 133         rc = dt_attr_get(env, o, &osi->osi_attr, NULL);
 134         if (rc)
 135                 GOTO(out, rc);
 136
 137         /* object will be released in device cleanup path */
 138         m->opd_last_used_file = o;
 139
 140         if (osi->osi_attr.la_size >= sizeof(osi->osi_id) *
 141                                      (m->opd_index + 1)) {
 142                 osp_objid_buf_prep(osi, m, m->opd_index);
 143                 rc = dt_record_read(env, o, &osi->osi_lb, &osi->osi_off);
 144                 if (rc != 0)
 145                         GOTO(out, rc);
 146         } else {
 147                 /* reset value to 0, just to make sure and change file's size */
 148                 struct thandle *th;
 149
 150                 m->opd_last_used_id = 0;
 151                 osp_objid_buf_prep(osi, m, m->opd_index);
 152
 153                 th = dt_trans_create(env, m->opd_storage);
 154                 if (IS_ERR(th))
 155                         GOTO(out, rc = PTR_ERR(th));
 156
 157                 rc = dt_declare_record_write(env, m->opd_last_used_file,
 158                                              osi->osi_lb.lb_len, osi->osi_off,
 159                                              th);
 160                 if (rc) {
 161                         dt_trans_stop(env, m->opd_storage, th);
 162                         GOTO(out, rc);
 163                 }
 164
 165                 rc = dt_trans_start_local(env, m->opd_storage, th);
 166                 if (rc) {
 167                         dt_trans_stop(env, m->opd_storage, th);
 168                         GOTO(out, rc);
 169                 }
 170
 171                 rc = dt_record_write(env, m->opd_last_used_file, &osi->osi_lb,
 172                                      &osi->osi_off, th);
 173                 dt_trans_stop(env, m->opd_storage, th);
 174                 if (rc)
 175                         GOTO(out, rc);
 176         }
 177         CDEBUG(D_HA, "%s: Read last used ID: "LPU64"\n", m->opd_obd->obd_name,
 178                le64_to_cpu(m->opd_last_used_id));
 179         RETURN(0);
 180 out:
 181         CERROR("%s: can't initialize lov_objid: %d\n",
 182                m->opd_obd->obd_name, rc);
 183         lu_object_put(env, &o->do_lu);
 184         m->opd_last_used_file = NULL;
 185         return rc;
 186 }
 187
 188 static void osp_last_used_fini(const struct lu_env *env, struct osp_device *d)
 189 {
 190         if (d->opd_last_used_file != NULL) {
 191                 lu_object_put(env, &d->opd_last_used_file->do_lu);
 192                 d->opd_last_used_file = NULL;
 193         }
 194 }
 195
 196 int osp_disconnect(struct osp_device *d)
 197 {
 198         struct obd_import *imp;
 199         int rc = 0;
 200
 201         imp = d->opd_obd->u.cli.cl_import;
 202
 203         /* Mark import deactivated now, so we don't try to reconnect if any
 204          * of the cleanup RPCs fails (e.g. ldlm cancel, etc).  We don't
 205          * fully deactivate the import, or that would drop all requests. */
 206         LASSERT(imp != NULL);
 207         spin_lock(&imp->imp_lock);
 208         imp->imp_deactive = 1;
 209         spin_unlock(&imp->imp_lock);
 210
 211         ptlrpc_deactivate_import(imp);
 212
 213         /* Some non-replayable imports (MDS's OSCs) are pinged, so just
 214          * delete it regardless.  (It's safe to delete an import that was
 215          * never added.) */
 216         (void)ptlrpc_pinger_del_import(imp);
 217
 218         rc = ptlrpc_disconnect_import(imp, 0);
 219         if (rc == -ETIMEDOUT || rc == -ENOTCONN || rc == -ESHUTDOWN)
 220                 rc = 0;
 221         if (rc)
 222                 CERROR("%s: can't disconnect: rc = %d\n",
 223                        d->opd_obd->obd_name, rc);
 224
 225         ptlrpc_invalidate_import(imp);
 226
 227         RETURN(rc);
 228 }
 229
 230 static int osp_shutdown(const struct lu_env *env, struct osp_device *d)
 231 {
 232         int                      rc = 0;
 233         ENTRY;
 234
 235         if (is_osp_for_connection(d->opd_obd->obd_name)) {
 236                 rc = osp_disconnect(d);
 237                 RETURN(rc);
 238         }
 239
 240         LASSERT(env);
 241         /* release last_used file */
 242         if (!d->opd_connect_mdt)
 243                 osp_last_used_fini(env, d);
 244
 245         rc = osp_disconnect(d);
 246
 247         if (!d->opd_connect_mdt) {
 248                 /* stop precreate thread */
 249                 osp_precreate_fini(d);
 250
 251                 /* stop sync thread */
 252                 osp_sync_fini(d);
 253         }
 254
 255         obd_fid_fini(d->opd_obd);
 256
 257         RETURN(rc);
 258 }
 259
 260 static int osp_process_config(const struct lu_env *env,
 261                               struct lu_device *dev, struct lustre_cfg *lcfg)
 262 {
 263         struct osp_device               *d = lu2osp_dev(dev);
 264         struct lprocfs_static_vars       lvars = { 0 };
 265         int                              rc;
 266
 267         ENTRY;
 268
 269         switch (lcfg->lcfg_command) {
 270         case LCFG_CLEANUP:
 271                 if (!is_osp_for_connection(d->opd_obd->obd_name))
 272                         lu_dev_del_linkage(dev->ld_site, dev);
 273                 rc = osp_shutdown(env, d);
 274                 break;
 275         case LCFG_PARAM:
 276                 lprocfs_osp_init_vars(&lvars);
 277
 278                 LASSERT(d->opd_obd);
 279                 rc = class_process_proc_param(PARAM_OSC, lvars.obd_vars,
 280                                               lcfg, d->opd_obd);
 281                 if (rc > 0)
 282                         rc = 0;
 283                 if (rc == -ENOSYS) {
 284                         /* class_process_proc_param() haven't found matching
 285                          * parameter and returned ENOSYS so that layer(s)
 286                          * below could use that. But OSP is the bottom, so
 287                          * just ignore it */
 288                         CERROR("%s: unknown param %s\n",
 289                                (char *)lustre_cfg_string(lcfg, 0),
 290                                (char *)lustre_cfg_string(lcfg, 1));
 291                         rc = 0;
 292                 }
 293                 break;
 294         default:
 295                 CERROR("%s: unknown command %u\n",
 296                        (char *)lustre_cfg_string(lcfg, 0), lcfg->lcfg_command);
 297                 rc = 0;
 298                 break;
 299         }
 300
 301         RETURN(rc);
 302 }
 303
 304 static int osp_recovery_complete(const struct lu_env *env,
 305                                  struct lu_device *dev)
 306 {
 307         struct osp_device       *osp = lu2osp_dev(dev);
 308         int                      rc = 0;
 309
 310         ENTRY;
 311         osp->opd_recovery_completed = 1;
 312         if (!osp->opd_connect_mdt)
 313                 cfs_waitq_signal(&osp->opd_pre_waitq);
 314         RETURN(rc);
 315 }
 316
 317 const struct lu_device_operations osp_lu_ops = {
 318         .ldo_object_alloc       = osp_object_alloc,
 319         .ldo_process_config     = osp_process_config,
 320         .ldo_recovery_complete  = osp_recovery_complete,
 321 };
 322
 323 /**
 324  * provides with statfs from corresponded OST
 325  *
 326  */
 327 static int osp_statfs(const struct lu_env *env, struct dt_device *dev,
 328                       struct obd_statfs *sfs)
 329 {
 330         struct osp_device *d = dt2osp_dev(dev);
 331
 332         ENTRY;
 333
 334         if (unlikely(d->opd_imp_active == 0))
 335                 RETURN(-ENOTCONN);
 336
 337         /* return recently updated data */
 338         *sfs = d->opd_statfs;
 339
 340         /*
 341          * layer above osp (usually lod) can use ffree to estimate
 342          * how many objects are available for immediate creation
 343          */
 344         spin_lock(&d->opd_pre_lock);
 345         sfs->os_fprecreated = d->opd_pre_last_created - d->opd_pre_used_id;
 346         sfs->os_fprecreated -= d->opd_pre_reserved;
 347         spin_unlock(&d->opd_pre_lock);
 348
 349         LASSERT(sfs->os_fprecreated <= OST_MAX_PRECREATE * 2);
 350
 351         CDEBUG(D_OTHER, "%s: "LPU64" blocks, "LPU64" free, "LPU64" avail, "
 352                LPU64" files, "LPU64" free files\n", d->opd_obd->obd_name,
 353                sfs->os_blocks, sfs->os_bfree, sfs->os_bavail,
 354                sfs->os_files, sfs->os_ffree);
 355         RETURN(0);
 356 }
 357
 358 static int osp_sync(const struct lu_env *env, struct dt_device *dev)
 359 {
 360         ENTRY;
 361
 362         /*
 363          * XXX: wake up sync thread, command it to start flushing asap?
 364          */
 365
 366         RETURN(0);
 367 }
 368
 369 const struct dt_device_operations osp_dt_ops = {
 370         .dt_statfs      = osp_statfs,
 371         .dt_sync        = osp_sync,
 372 };
 373
 374 static int osp_connect_to_osd(const struct lu_env *env, struct osp_device *m,
 375                               const char *nextdev)
 376 {
 377         struct obd_connect_data *data = NULL;
 378         struct obd_device       *obd;
 379         int                      rc;
 380
 381         ENTRY;
 382
 383         LASSERT(m->opd_storage_exp == NULL);
 384
 385         OBD_ALLOC_PTR(data);
 386         if (data == NULL)
 387                 RETURN(-ENOMEM);
 388
 389         obd = class_name2obd(nextdev);
 390         if (obd == NULL) {
 391                 CERROR("%s: can't locate next device: %s\n",
 392                        m->opd_obd->obd_name, nextdev);
 393                 GOTO(out, rc = -ENOTCONN);
 394         }
 395
 396         rc = obd_connect(env, &m->opd_storage_exp, obd, &obd->obd_uuid, data,
 397                          NULL);
 398         if (rc) {
 399                 CERROR("%s: cannot connect to next dev %s: rc = %d\n",
 400                        m->opd_obd->obd_name, nextdev, rc);
 401                 GOTO(out, rc);
 402         }
 403
 404         m->opd_dt_dev.dd_lu_dev.ld_site =
 405                 m->opd_storage_exp->exp_obd->obd_lu_dev->ld_site;
 406         LASSERT(m->opd_dt_dev.dd_lu_dev.ld_site);
 407         m->opd_storage = lu2dt_dev(m->opd_storage_exp->exp_obd->obd_lu_dev);
 408
 409 out:
 410         OBD_FREE_PTR(data);
 411         RETURN(rc);
 412 }
 413
 414 static int osp_init0(const struct lu_env *env, struct osp_device *m,
 415                      struct lu_device_type *ldt, struct lustre_cfg *cfg)
 416 {
 417         struct obd_device       *obd;
 418         struct obd_import       *imp;
 419         class_uuid_t            uuid;
 420         char                    *src, *tgt, *mdt, *osdname = NULL;
 421         int                     rc, idx;
 422
 423         ENTRY;
 424
 425         obd = class_name2obd(lustre_cfg_string(cfg, 0));
 426         if (obd == NULL) {
 427                 CERROR("Cannot find obd with name %s\n",
 428                        lustre_cfg_string(cfg, 0));
 429                 RETURN(-ENODEV);
 430         }
 431         m->opd_obd = obd;
 432
 433         /* There is no record in the MDT configuration for the local disk
 434          * device, so we have to extract this from elsewhere in the profile.
 435          * The only information we get at setup is from the OSC records:
 436          * setup 0:{fsname}-OSTxxxx-osc[-MDTxxxx] 1:lustre-OST0000_UUID 2:NID
 437          * Note that 1.8 generated configs are missing the -MDTxxxx part.
 438          * We need to reconstruct the name of the underlying OSD from this:
 439          * {fsname}-{svname}-osd, for example "lustre-MDT0000-osd".  We
 440          * also need to determine the OST index from this - will be used
 441          * to calculate the offset in shared lov_objids file later */
 442
 443         src = lustre_cfg_string(cfg, 0);
 444         if (src == NULL)
 445                 RETURN(-EINVAL);
 446
 447         tgt = strrchr(src, '-');
 448         if (tgt == NULL) {
 449                 CERROR("%s: invalid target name %s\n",
 450                        m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
 451                 RETURN(-EINVAL);
 452         }
 453
 454         if (strncmp(tgt, "-osc", 4) == 0) {
 455                 /* Old OSC name fsname-OSTXXXX-osc */
 456                 for (tgt--; tgt > src && *tgt != '-'; tgt--)
 457                         ;
 458                 if (tgt == src) {
 459                         CERROR("%s: invalid target name %s\n",
 460                                m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
 461                         RETURN(-EINVAL);
 462                 }
 463
 464                 if (strncmp(tgt, "-OST", 4) != 0) {
 465                         CERROR("%s: invalid target name %s\n",
 466                                m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
 467                         RETURN(-EINVAL);
 468                 }
 469
 470                 idx = simple_strtol(tgt + 4, &mdt, 16);
 471                 if (mdt[0] != '-' || idx > INT_MAX || idx < 0) {
 472                         CERROR("%s: invalid OST index in '%s'\n",
 473                                m->opd_obd->obd_name, src);
 474                         RETURN(-EINVAL);
 475                 }
 476                 m->opd_index = idx;
 477                 idx = tgt - src;
 478         } else {
 479                 /* New OSC name fsname-OSTXXXX-osc-MDTXXXX */
 480                 if (strncmp(tgt, "-MDT", 4) != 0 &&
 481                          strncmp(tgt, "-OST", 4) != 0) {
 482                         CERROR("%s: invalid target name %s\n",
 483                                m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
 484                         RETURN(-EINVAL);
 485                 }
 486
 487                 if (tgt - src <= 12) {
 488                         CERROR("%s: invalid target name %s\n",
 489                                m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
 490                         RETURN(-EINVAL);
 491                 }
 492
 493                 if (strncmp(tgt - 12, "-MDT", 4) == 0)
 494                         m->opd_connect_mdt = 1;
 495
 496                 idx = simple_strtol(tgt - 8, &mdt, 16);
 497                 if (mdt[0] != '-' || idx > INT_MAX || idx < 0) {
 498                         CERROR("%s: invalid OST index in '%s'\n",
 499                                m->opd_obd->obd_name, src);
 500                         RETURN(-EINVAL);
 501                 }
 502
 503                 m->opd_index = idx;
 504                 idx = tgt - src - 12;
 505         }
 506         /* check the fsname length, and after this everything else will fit */
 507         if (idx > MTI_NAME_MAXLEN) {
 508                 CERROR("%s: fsname too long in '%s'\n",
 509                        m->opd_obd->obd_name, src);
 510                 RETURN(-EINVAL);
 511         }
 512
 513         OBD_ALLOC(osdname, MAX_OBD_NAME);
 514         if (osdname == NULL)
 515                 RETURN(-ENOMEM);
 516
 517         memcpy(osdname, src, idx); /* copy just the fsname part */
 518         osdname[idx] = '\0';
 519
 520         mdt = strstr(mdt, "-MDT");
 521         if (mdt == NULL) /* 1.8 configs don't have "-MDT0000" at the end */
 522                 strcat(osdname, "-MDT0000");
 523         else
 524                 strcat(osdname, mdt);
 525         strcat(osdname, "-osd");
 526         CDEBUG(D_HA, "%s: connect to %s (%s)\n", obd->obd_name, osdname, src);
 527
 528         if (m->opd_connect_mdt) {
 529                 struct client_obd *cli = &m->opd_obd->u.cli;
 530
 531                 OBD_ALLOC(cli->cl_rpc_lock, sizeof(*cli->cl_rpc_lock));
 532                 if (!cli->cl_rpc_lock)
 533                         RETURN(-ENOMEM);
 534                 osp_init_rpc_lock(cli->cl_rpc_lock);
 535         }
 536
 537         m->opd_dt_dev.dd_lu_dev.ld_ops = &osp_lu_ops;
 538         m->opd_dt_dev.dd_ops = &osp_dt_ops;
 539         obd->obd_lu_dev = &m->opd_dt_dev.dd_lu_dev;
 540
 541         rc = osp_connect_to_osd(env, m, osdname);
 542         if (rc)
 543                 GOTO(out_fini, rc);
 544
 545         rc = ptlrpcd_addref();
 546         if (rc)
 547                 GOTO(out_disconnect, rc);
 548
 549         rc = client_obd_setup(obd, cfg);
 550         if (rc) {
 551                 CERROR("%s: can't setup obd: %d\n", m->opd_obd->obd_name, rc);
 552                 GOTO(out_ref, rc);
 553         }
 554
 555         osp_lprocfs_init(m);
 556
 557         if (!m->opd_connect_mdt) {
 558                 /* Initialize last id from the storage - will be
 559                  * used in orphan cleanup. */
 560                 rc = osp_last_used_init(env, m);
 561                 if (rc)
 562                         GOTO(out_proc, rc);
 563                 /* Initialize precreation thread, it handles new
 564                  * connections as well. */
 565                 rc = osp_init_precreate(m);
 566                 if (rc)
 567                         GOTO(out_last_used, rc);
 568                 /*
 569                  * Initialize synhronization mechanism taking
 570                  * care of propogating changes to OST in near
 571                  * transactional manner.
 572                  */
 573                 rc = osp_sync_init(env, m);
 574                 if (rc)
 575                         GOTO(out_precreat, rc);
 576
 577                 rc = obd_fid_init(m->opd_obd, NULL, LUSTRE_SEQ_DATA);
 578                 if (rc) {
 579                         CERROR("%s: fid init error: rc = %d\n",
 580                                m->opd_obd->obd_name, rc);
 581                         GOTO(out, rc);
 582                 }
 583         }
 584         /*
 585          * Initiate connect to OST
 586          */
 587         ll_generate_random_uuid(uuid);
 588         class_uuid_unparse(uuid, &m->opd_cluuid);
 589
 590         imp = obd->u.cli.cl_import;
 591
 592         rc = ptlrpc_init_import(imp);
 593         if (rc)
 594                 GOTO(out, rc);
 595         if (osdname)
 596                 OBD_FREE(osdname, MAX_OBD_NAME);
 597         RETURN(0);
 598
 599 out:
 600         if (!m->opd_connect_mdt)
 601                 /* stop sync thread */
 602                 osp_sync_fini(m);
 603 out_precreat:
 604         /* stop precreate thread */
 605         if (!m->opd_connect_mdt)
 606                 osp_precreate_fini(m);
 607 out_last_used:
 608         osp_last_used_fini(env, m);
 609 out_proc:
 610         ptlrpc_lprocfs_unregister_obd(obd);
 611         lprocfs_obd_cleanup(obd);
 612         class_destroy_import(obd->u.cli.cl_import);
 613         client_obd_cleanup(obd);
 614 out_ref:
 615         ptlrpcd_decref();
 616 out_disconnect:
 617         if (m->opd_connect_mdt) {
 618                 struct client_obd *cli = &m->opd_obd->u.cli;
 619                 if (cli->cl_rpc_lock != NULL) {
 620                         OBD_FREE_PTR(cli->cl_rpc_lock);
 621                         cli->cl_rpc_lock = NULL;
 622                 }
 623         }
 624         obd_disconnect(m->opd_storage_exp);
 625 out_fini:
 626         if (osdname)
 627                 OBD_FREE(osdname, MAX_OBD_NAME);
 628         RETURN(rc);
 629 }
 630
 631 static struct lu_device *osp_device_free(const struct lu_env *env,
 632                                          struct lu_device *lu)
 633 {
 634         struct osp_device *m = lu2osp_dev(lu);
 635
 636         ENTRY;
 637
 638         if (cfs_atomic_read(&lu->ld_ref) && lu->ld_site) {
 639                 LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);
 640                 lu_site_print(env, lu->ld_site, &msgdata, lu_cdebug_printer);
 641         }
 642         dt_device_fini(&m->opd_dt_dev);
 643         OBD_FREE_PTR(m);
 644         RETURN(NULL);
 645 }
 646
 647 static struct lu_device *osp_device_alloc(const struct lu_env *env,
 648                                           struct lu_device_type *t,
 649                                           struct lustre_cfg *lcfg)
 650 {
 651         struct osp_device *m;
 652         struct lu_device  *l;
 653
 654         OBD_ALLOC_PTR(m);
 655         if (m == NULL) {
 656                 l = ERR_PTR(-ENOMEM);
 657         } else {
 658                 int rc;
 659
 660                 l = osp2lu_dev(m);
 661                 dt_device_init(&m->opd_dt_dev, t);
 662                 if (is_osp_for_connection(lustre_cfg_string(lcfg, 0)))
 663                         rc = osp_init_for_ost(env, m, t, lcfg);
 664                 else
 665                         rc = osp_init0(env, m, t, lcfg);
 666                 if (rc != 0) {
 667                         osp_device_free(env, l);
 668                         l = ERR_PTR(rc);
 669                 }
 670         }
 671         return l;
 672 }
 673
 674 static struct lu_device *osp_device_fini(const struct lu_env *env,
 675                                          struct lu_device *d)
 676 {
 677         struct osp_device *m = lu2osp_dev(d);
 678         struct obd_import *imp;
 679         int                rc;
 680
 681         ENTRY;
 682
 683         if (m->opd_storage_exp)
 684                 obd_disconnect(m->opd_storage_exp);
 685
 686         if (is_osp_for_connection(m->opd_obd->obd_name))
 687                 osp_fini_for_ost(m);
 688
 689         imp = m->opd_obd->u.cli.cl_import;
 690
 691         if (imp->imp_rq_pool) {
 692                 ptlrpc_free_rq_pool(imp->imp_rq_pool);
 693                 imp->imp_rq_pool = NULL;
 694         }
 695
 696         obd_cleanup_client_import(m->opd_obd);
 697
 698         if (m->opd_symlink)
 699                 lprocfs_remove(&m->opd_symlink);
 700
 701         LASSERT(m->opd_obd);
 702         ptlrpc_lprocfs_unregister_obd(m->opd_obd);
 703         lprocfs_obd_cleanup(m->opd_obd);
 704
 705         if (m->opd_connect_mdt) {
 706                 struct client_obd *cli = &m->opd_obd->u.cli;
 707                 if (cli->cl_rpc_lock != NULL) {
 708                         OBD_FREE_PTR(cli->cl_rpc_lock);
 709                         cli->cl_rpc_lock = NULL;
 710                 }
 711         }
 712
 713         rc = client_obd_cleanup(m->opd_obd);
 714         LASSERTF(rc == 0, "error %d\n", rc);
 715
 716         ptlrpcd_decref();
 717
 718         RETURN(NULL);
 719 }
 720
 721 static int osp_reconnect(const struct lu_env *env,
 722                          struct obd_export *exp, struct obd_device *obd,
 723                          struct obd_uuid *cluuid,
 724                          struct obd_connect_data *data,
 725                          void *localdata)
 726 {
 727         return 0;
 728 }
 729
 730 /*
 731  * we use exports to track all LOD users
 732  */
 733 static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp,
 734                            struct obd_device *obd, struct obd_uuid *cluuid,
 735                            struct obd_connect_data *data, void *localdata)
 736 {
 737         struct osp_device       *osp = lu2osp_dev(obd->obd_lu_dev);
 738         struct obd_connect_data *ocd;
 739         struct obd_import       *imp;
 740         struct lustre_handle     conn;
 741         int                      rc;
 742
 743         ENTRY;
 744
 745         CDEBUG(D_CONFIG, "connect #%d\n", osp->opd_connects);
 746
 747         rc = class_connect(&conn, obd, cluuid);
 748         if (rc)
 749                 RETURN(rc);
 750
 751         *exp = class_conn2export(&conn);
 752         osp->opd_exp = *exp;
 753
 754         /* Why should there ever be more than 1 connect? */
 755         osp->opd_connects++;
 756         LASSERT(osp->opd_connects == 1);
 757
 758         osp->opd_exp = *exp;
 759
 760         imp = osp->opd_obd->u.cli.cl_import;
 761         imp->imp_dlm_handle = conn;
 762
 763         LASSERT(data != NULL);
 764         LASSERT(data->ocd_connect_flags & OBD_CONNECT_INDEX);
 765         ocd = &imp->imp_connect_data;
 766         *ocd = *data;
 767         if (is_osp_for_connection(osp->opd_obd->obd_name))
 768                 ocd->ocd_connect_flags |= OBD_CONNECT_LIGHTWEIGHT;
 769
 770         imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
 771
 772         ocd->ocd_version = LUSTRE_VERSION_CODE;
 773         ocd->ocd_index = data->ocd_index;
 774         imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
 775
 776         rc = ptlrpc_connect_import(imp);
 777         if (rc) {
 778                 CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc);
 779                 GOTO(out, rc);
 780         }
 781
 782         ptlrpc_pinger_add_import(imp);
 783
 784         /* set seq controller export for MDC0 if exists */
 785         if (osp->opd_connect_mdt && !is_osp_for_connection(obd->obd_name) &&
 786             data->ocd_index == 0) {
 787                 struct seq_server_site *ss;
 788
 789                 ss = lu_site2seq(osp2lu_dev(osp)->ld_site);
 790                 ss->ss_control_exp = class_export_get(*exp);
 791                 ss->ss_server_fld->lsf_control_exp = *exp;
 792         }
 793
 794 out:
 795         RETURN(rc);
 796 }
 797
 798 /*
 799  * once last export (we don't count self-export) disappeared
 800  * osp can be released
 801  */
 802 static int osp_obd_disconnect(struct obd_export *exp)
 803 {
 804         struct obd_device *obd = exp->exp_obd;
 805         struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev);
 806         int                rc;
 807         ENTRY;
 808
 809         /* Only disconnect the underlying layers on the final disconnect. */
 810         LASSERT(osp->opd_connects == 1);
 811         osp->opd_connects--;
 812
 813         rc = class_disconnect(exp);
 814         if (rc) {
 815                 CERROR("%s: class disconnect error: rc = %d\n",
 816                        obd->obd_name, rc);
 817                 RETURN(rc);
 818         }
 819
 820         /* destroy the device */
 821         if (!is_osp_for_connection(obd->obd_name))
 822                 class_manual_cleanup(obd);
 823
 824         RETURN(rc);
 825 }
 826
 827 /*
 828  * lprocfs helpers still use OBD API, let's keep obd_statfs() support
 829  */
 830 static int osp_obd_statfs(const struct lu_env *env, struct obd_export *exp,
 831                           struct obd_statfs *osfs, __u64 max_age, __u32 flags)
 832 {
 833         struct obd_statfs       *msfs;
 834         struct ptlrpc_request   *req;
 835         struct obd_import       *imp = NULL;
 836         int                      rc;
 837
 838         ENTRY;
 839
 840         /* Since the request might also come from lprocfs, so we need
 841          * sync this with client_disconnect_export Bug15684 */
 842         down_read(&exp->exp_obd->u.cli.cl_sem);
 843         if (exp->exp_obd->u.cli.cl_import)
 844                 imp = class_import_get(exp->exp_obd->u.cli.cl_import);
 845         up_read(&exp->exp_obd->u.cli.cl_sem);
 846         if (!imp)
 847                 RETURN(-ENODEV);
 848
 849         /* We could possibly pass max_age in the request (as an absolute
 850          * timestamp or a "seconds.usec ago") so the target can avoid doing
 851          * extra calls into the filesystem if that isn't necessary (e.g.
 852          * during mount that would help a bit).  Having relative timestamps
 853          * is not so great if request processing is slow, while absolute
 854          * timestamps are not ideal because they need time synchronization. */
 855         req = ptlrpc_request_alloc(imp, &RQF_OST_STATFS);
 856
 857         class_import_put(imp);
 858
 859         if (req == NULL)
 860                 RETURN(-ENOMEM);
 861
 862         rc = ptlrpc_request_pack(req, LUSTRE_OST_VERSION, OST_STATFS);
 863         if (rc) {
 864                 ptlrpc_request_free(req);
 865                 RETURN(rc);
 866         }
 867         ptlrpc_request_set_replen(req);
 868         req->rq_request_portal = OST_CREATE_PORTAL;
 869         ptlrpc_at_set_req_timeout(req);
 870
 871         if (flags & OBD_STATFS_NODELAY) {
 872                 /* procfs requests not want stat in wait for avoid deadlock */
 873                 req->rq_no_resend = 1;
 874                 req->rq_no_delay = 1;
 875         }
 876
 877         rc = ptlrpc_queue_wait(req);
 878         if (rc)
 879                 GOTO(out, rc);
 880
 881         msfs = req_capsule_server_get(&req->rq_pill, &RMF_OBD_STATFS);
 882         if (msfs == NULL)
 883                 GOTO(out, rc = -EPROTO);
 884
 885         *osfs = *msfs;
 886
 887         EXIT;
 888 out:
 889         ptlrpc_req_finished(req);
 890         return rc;
 891 }
 892
 893 static int osp_import_event(struct obd_device *obd, struct obd_import *imp,
 894                             enum obd_import_event event)
 895 {
 896         struct osp_device *d = lu2osp_dev(obd->obd_lu_dev);
 897
 898         switch (event) {
 899         case IMP_EVENT_DISCON:
 900                 d->opd_got_disconnected = 1;
 901                 d->opd_imp_connected = 0;
 902                 if (d->opd_connect_mdt)
 903                         break;
 904                 osp_pre_update_status(d, -ENODEV);
 905                 cfs_waitq_signal(&d->opd_pre_waitq);
 906                 CDEBUG(D_HA, "got disconnected\n");
 907                 break;
 908         case IMP_EVENT_INACTIVE:
 909                 d->opd_imp_active = 0;
 910                 if (d->opd_connect_mdt)
 911                         break;
 912                 osp_pre_update_status(d, -ENODEV);
 913                 cfs_waitq_signal(&d->opd_pre_waitq);
 914                 CDEBUG(D_HA, "got inactive\n");
 915                 break;
 916         case IMP_EVENT_ACTIVE:
 917                 d->opd_imp_active = 1;
 918                 if (d->opd_got_disconnected)
 919                         d->opd_new_connection = 1;
 920                 d->opd_imp_connected = 1;
 921                 d->opd_imp_seen_connected = 1;
 922                 if (d->opd_connect_mdt)
 923                         break;
 924                 if (d->opd_obd->u.cli.cl_seq->lcs_exp == NULL)
 925                         d->opd_obd->u.cli.cl_seq->lcs_exp =
 926                                         class_export_get(d->opd_exp);
 927                 cfs_waitq_signal(&d->opd_pre_waitq);
 928                 __osp_sync_check_for_work(d);
 929                 CDEBUG(D_HA, "got connected\n");
 930                 break;
 931         case IMP_EVENT_INVALIDATE:
 932                 if (obd->obd_namespace == NULL)
 933                         break;
 934                 ldlm_namespace_cleanup(obd->obd_namespace, LDLM_FL_LOCAL_ONLY);
 935                 break;
 936         case IMP_EVENT_OCD:
 937         case IMP_EVENT_DEACTIVATE:
 938         case IMP_EVENT_ACTIVATE:
 939                 break;
 940         default:
 941                 CERROR("%s: unsupported import event: %#x\n",
 942                        obd->obd_name, event);
 943         }
 944         return 0;
 945 }
 946
 947 static int osp_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
 948                          void *karg, void *uarg)
 949 {
 950         struct obd_device       *obd = exp->exp_obd;
 951         struct osp_device       *d;
 952         struct obd_ioctl_data   *data = karg;
 953         int                      rc = 0;
 954
 955         ENTRY;
 956
 957         LASSERT(obd->obd_lu_dev);
 958         d = lu2osp_dev(obd->obd_lu_dev);
 959         LASSERT(d->opd_dt_dev.dd_ops == &osp_dt_ops);
 960
 961         if (!cfs_try_module_get(THIS_MODULE)) {
 962                 CERROR("%s: can't get module. Is it alive?", obd->obd_name);
 963                 return -EINVAL;
 964         }
 965
 966         switch (cmd) {
 967         case OBD_IOC_CLIENT_RECOVER:
 968                 rc = ptlrpc_recover_import(obd->u.cli.cl_import,
 969                                            data->ioc_inlbuf1, 0);
 970                 if (rc > 0)
 971                         rc = 0;
 972                 break;
 973         case IOC_OSC_SET_ACTIVE:
 974                 rc = ptlrpc_set_import_active(obd->u.cli.cl_import,
 975                                               data->ioc_offset);
 976                 break;
 977         case OBD_IOC_PING_TARGET:
 978                 rc = ptlrpc_obd_ping(obd);
 979                 break;
 980         default:
 981                 CERROR("%s: unrecognized ioctl %#x by %s\n", obd->obd_name,
 982                        cmd, cfs_curproc_comm());
 983                 rc = -ENOTTY;
 984         }
 985         cfs_module_put(THIS_MODULE);
 986         return rc;
 987 }
 988
 989 static int osp_obd_health_check(const struct lu_env *env,
 990                                 struct obd_device *obd)
 991 {
 992         struct osp_device *d = lu2osp_dev(obd->obd_lu_dev);
 993
 994         ENTRY;
 995
 996         /*
 997          * 1.8/2.0 behaviour is that OST being connected once at least
 998          * is considired "healthy". and one "healty" OST is enough to
 999          * allow lustre clients to connect to MDS
1000          */
1001         LASSERT(d);
1002         RETURN(!d->opd_imp_seen_connected);
1003 }
1004
1005 /* context key constructor/destructor: mdt_key_init, mdt_key_fini */
1006 LU_KEY_INIT_FINI(osp, struct osp_thread_info);
1007 static void osp_key_exit(const struct lu_context *ctx,
1008                          struct lu_context_key *key, void *data)
1009 {
1010         struct osp_thread_info *info = data;
1011
1012         info->osi_attr.la_valid = 0;
1013 }
1014
1015 struct lu_context_key osp_thread_key = {
1016         .lct_tags = LCT_MD_THREAD,
1017         .lct_init = osp_key_init,
1018         .lct_fini = osp_key_fini,
1019         .lct_exit = osp_key_exit
1020 };
1021
1022 /* context key constructor/destructor: mdt_txn_key_init, mdt_txn_key_fini */
1023 LU_KEY_INIT_FINI(osp_txn, struct osp_txn_info);
1024
1025 struct lu_context_key osp_txn_key = {
1026         .lct_tags = LCT_OSP_THREAD,
1027         .lct_init = osp_txn_key_init,
1028         .lct_fini = osp_txn_key_fini
1029 };
1030 LU_TYPE_INIT_FINI(osp, &osp_thread_key, &osp_txn_key);
1031
1032 static struct lu_device_type_operations osp_device_type_ops = {
1033         .ldto_init           = osp_type_init,
1034         .ldto_fini           = osp_type_fini,
1035
1036         .ldto_start          = osp_type_start,
1037         .ldto_stop           = osp_type_stop,
1038
1039         .ldto_device_alloc   = osp_device_alloc,
1040         .ldto_device_free    = osp_device_free,
1041
1042         .ldto_device_fini    = osp_device_fini
1043 };
1044
1045 static struct lu_device_type osp_device_type = {
1046         .ldt_tags     = LU_DEVICE_DT,
1047         .ldt_name     = LUSTRE_OSP_NAME,
1048         .ldt_ops      = &osp_device_type_ops,
1049         .ldt_ctx_tags = LCT_MD_THREAD
1050 };
1051
1052 static struct obd_ops osp_obd_device_ops = {
1053         .o_owner        = THIS_MODULE,
1054         .o_add_conn     = client_import_add_conn,
1055         .o_del_conn     = client_import_del_conn,
1056         .o_reconnect    = osp_reconnect,
1057         .o_connect      = osp_obd_connect,
1058         .o_disconnect   = osp_obd_disconnect,
1059         .o_health_check = osp_obd_health_check,
1060         .o_import_event = osp_import_event,
1061         .o_iocontrol    = osp_iocontrol,
1062         .o_statfs       = osp_obd_statfs,
1063         .o_fid_init     = client_fid_init,
1064         .o_fid_fini     = client_fid_fini,
1065 };
1066
1067 struct llog_operations osp_mds_ost_orig_logops;
1068
1069 static int __init osp_mod_init(void)
1070 {
1071         struct lprocfs_static_vars       lvars;
1072         cfs_proc_dir_entry_t            *osc_proc_dir;
1073         int                              rc;
1074
1075         rc = lu_kmem_init(osp_caches);
1076         if (rc)
1077                 return rc;
1078
1079         lprocfs_osp_init_vars(&lvars);
1080
1081         rc = class_register_type(&osp_obd_device_ops, NULL, lvars.module_vars,
1082                                  LUSTRE_OSP_NAME, &osp_device_type);
1083
1084         /* create "osc" entry in procfs for compatibility purposes */
1085         if (rc != 0) {
1086                 lu_kmem_fini(osp_caches);
1087                 return rc;
1088         }
1089
1090         /* Note: add_rec/delcare_add_rec will be only used by catalogs */
1091         osp_mds_ost_orig_logops = llog_osd_ops;
1092         osp_mds_ost_orig_logops.lop_add = llog_cat_add_rec;
1093         osp_mds_ost_orig_logops.lop_declare_add = llog_cat_declare_add_rec;
1094
1095         osc_proc_dir = lprocfs_srch(proc_lustre_root, "osc");
1096         if (osc_proc_dir == NULL) {
1097                 osc_proc_dir = lprocfs_register("osc", proc_lustre_root, NULL,
1098                                                 NULL);
1099                 if (IS_ERR(osc_proc_dir))
1100                         CERROR("osp: can't create compat entry \"osc\": %d\n",
1101                                (int) PTR_ERR(osc_proc_dir));
1102         }
1103         return rc;
1104 }
1105
1106 static void __exit osp_mod_exit(void)
1107 {
1108         lprocfs_try_remove_proc_entry("osc", proc_lustre_root);
1109
1110         class_unregister_type(LUSTRE_OSP_NAME);
1111         lu_kmem_fini(osp_caches);
1112 }
1113
1114 MODULE_AUTHOR("Intel, Inc. <http://www.intel.com/>");
1115 MODULE_DESCRIPTION("Lustre OST Proxy Device ("LUSTRE_OSP_NAME")");
1116 MODULE_LICENSE("GPL");
1117
1118 cfs_module(osp, LUSTRE_VERSION_STRING, osp_mod_init, osp_mod_exit);