4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; If not, see
18 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
20 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21 * CA 95054 USA or visit www.sun.com if you need additional information or
27 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Use is subject to license terms.
30 * Copyright (c) 2012, Intel Corporation.
33 * This file is part of Lustre, http://www.lustre.org/
34 * Lustre is a trademark of Sun Microsystems, Inc.
36 * lustre/osp/osp_dev.c
38 * Lustre OST Proxy Device
40 * Author: Alex Zhuravlev <alexey.zhuravlev@intel.com>
41 * Author: Mikhail Pershin <mike.pershin@intel.com>
42 * Author: Di Wang <di.wang@intel.com>
46 # define EXPORT_SYMTAB
48 #define DEBUG_SUBSYSTEM S_MDS
50 #include <obd_class.h>
51 #include <lustre_param.h>
52 #include <lustre_log.h>
53 #include <lustre_mdc.h>
55 #include "osp_internal.h"
57 /* Slab for OSP object allocation */
58 cfs_mem_cache_t *osp_object_kmem;
60 static struct lu_kmem_descr osp_caches[] = {
62 .ckd_cache = &osp_object_kmem,
63 .ckd_name = "osp_obj",
64 .ckd_size = sizeof(struct osp_object)
71 struct lu_object *osp_object_alloc(const struct lu_env *env,
72 const struct lu_object_header *hdr,
75 struct lu_object_header *h = NULL;
79 OBD_SLAB_ALLOC_PTR_GFP(o, osp_object_kmem, CFS_ALLOC_IO);
81 l = &o->opo_obj.do_lu;
83 /* For data object, OSP obj would always be the top
84 * object, i.e. hdr is always NULL, see lu_object_alloc.
85 * But for metadata object, we always build the object
86 * stack from MDT. i.e. mdt_object will be the top object
91 lu_object_header_init(h);
92 dt_object_init(&o->opo_obj, h, d);
93 lu_object_add_top(h, l);
95 dt_object_init(&o->opo_obj, h, d);
98 l->lo_ops = &osp_lu_obj_ops;
106 static struct dt_object
107 *osp_find_or_create(const struct lu_env *env, struct osp_device *osp,
108 struct lu_attr *attr, __u32 reg_id)
110 struct osp_thread_info *osi = osp_env_info(env);
111 struct dt_object_format dof = { 0 };
112 struct dt_object *dto;
116 lu_local_obj_fid(&osi->osi_fid, reg_id);
117 attr->la_valid = LA_MODE;
118 attr->la_mode = S_IFREG | 0644;
119 dof.dof_type = DFT_REGULAR;
120 dto = dt_find_or_create(env, osp->opd_storage, &osi->osi_fid,
125 rc = dt_attr_get(env, dto, attr, NULL);
127 CERROR("%s: can't be initialized: rc = %d\n",
128 osp->opd_obd->obd_name, rc);
129 lu_object_put(env, &dto->do_lu);
135 static int osp_write_local_file(const struct lu_env *env,
136 struct osp_device *osp,
137 struct dt_object *dt_obj,
144 th = dt_trans_create(env, osp->opd_storage);
148 rc = dt_declare_record_write(env, dt_obj, buf->lb_len, offset, th);
151 rc = dt_trans_start_local(env, osp->opd_storage, th);
155 rc = dt_record_write(env, dt_obj, buf, &offset, th);
157 dt_trans_stop(env, osp->opd_storage, th);
161 static int osp_init_last_objid(const struct lu_env *env, struct osp_device *osp)
163 struct osp_thread_info *osi = osp_env_info(env);
164 struct lu_fid *fid = &osp->opd_last_used_fid;
165 struct dt_object *dto;
169 dto = osp_find_or_create(env, osp, &osi->osi_attr, MDD_LOV_OBJ_OID);
171 RETURN(PTR_ERR(dto));
172 /* object will be released in device cleanup path */
173 if (osi->osi_attr.la_size >=
174 sizeof(osi->osi_id) * (osp->opd_index + 1)) {
175 osp_objid_buf_prep(&osi->osi_lb, &osi->osi_off, &fid->f_oid,
177 rc = dt_record_read(env, dto, &osi->osi_lb, &osi->osi_off);
182 osp_objid_buf_prep(&osi->osi_lb, &osi->osi_off, &fid->f_oid,
184 rc = osp_write_local_file(env, osp, dto, &osi->osi_lb,
187 osp->opd_last_used_oid_file = dto;
190 /* object will be released in device cleanup path */
191 CERROR("%s: can't initialize lov_objid: rc = %d\n",
192 osp->opd_obd->obd_name, rc);
193 lu_object_put(env, &dto->do_lu);
194 osp->opd_last_used_oid_file = NULL;
198 static int osp_init_last_seq(const struct lu_env *env, struct osp_device *osp)
200 struct osp_thread_info *osi = osp_env_info(env);
201 struct lu_fid *fid = &osp->opd_last_used_fid;
202 struct dt_object *dto;
206 dto = osp_find_or_create(env, osp, &osi->osi_attr, MDD_LOV_OBJ_OSEQ);
208 RETURN(PTR_ERR(dto));
210 /* object will be released in device cleanup path */
211 if (osi->osi_attr.la_size >=
212 sizeof(osi->osi_id) * (osp->opd_index + 1)) {
213 osp_objseq_buf_prep(&osi->osi_lb, &osi->osi_off, &fid->f_seq,
215 rc = dt_record_read(env, dto, &osi->osi_lb, &osi->osi_off);
220 osp_objseq_buf_prep(&osi->osi_lb, &osi->osi_off, &fid->f_seq,
222 rc = osp_write_local_file(env, osp, dto, &osi->osi_lb,
225 osp->opd_last_used_seq_file = dto;
228 /* object will be released in device cleanup path */
229 CERROR("%s: can't initialize lov_seq: rc = %d\n",
230 osp->opd_obd->obd_name, rc);
231 lu_object_put(env, &dto->do_lu);
232 osp->opd_last_used_seq_file = NULL;
236 static int osp_last_used_init(const struct lu_env *env, struct osp_device *osp)
238 struct osp_thread_info *osi = osp_env_info(env);
242 fid_zero(&osp->opd_last_used_fid);
243 rc = osp_init_last_objid(env, osp);
245 CERROR("%s: Can not get ids %d from old objid!\n",
246 osp->opd_obd->obd_name, rc);
250 rc = osp_init_last_seq(env, osp);
252 CERROR("%s: Can not get ids %d from old objid!\n",
253 osp->opd_obd->obd_name, rc);
257 if (fid_oid(&osp->opd_last_used_fid) != 0 &&
258 fid_seq(&osp->opd_last_used_fid) == 0) {
259 /* Just upgrade from the old version,
260 * set the seq to be IDIF */
261 osp->opd_last_used_fid.f_seq =
262 fid_idif_seq(fid_oid(&osp->opd_last_used_fid),
264 osp_objseq_buf_prep(&osi->osi_lb, &osi->osi_off,
265 &osp->opd_last_used_fid.f_seq,
267 rc = osp_write_local_file(env, osp, osp->opd_last_used_seq_file,
268 &osi->osi_lb, osi->osi_off);
270 CERROR("%s : Can not write seq file: rc = %d\n",
271 osp->opd_obd->obd_name, rc);
276 if (!fid_is_zero(&osp->opd_last_used_fid) &&
277 !fid_is_sane(&osp->opd_last_used_fid)) {
278 CERROR("%s: Got invalid FID "DFID"\n", osp->opd_obd->obd_name,
279 PFID(&osp->opd_last_used_fid));
280 GOTO(out, rc = -EINVAL);
283 CDEBUG(D_INFO, "%s: Init last used fid "DFID"\n",
284 osp->opd_obd->obd_name, PFID(&osp->opd_last_used_fid));
287 if (osp->opd_last_used_oid_file != NULL) {
288 lu_object_put(env, &osp->opd_last_used_oid_file->do_lu);
289 osp->opd_last_used_oid_file = NULL;
291 if (osp->opd_last_used_seq_file != NULL) {
292 lu_object_put(env, &osp->opd_last_used_seq_file->do_lu);
293 osp->opd_last_used_seq_file = NULL;
300 static void osp_last_used_fini(const struct lu_env *env, struct osp_device *d)
302 /* release last_used file */
303 if (d->opd_last_used_oid_file != NULL) {
304 lu_object_put(env, &d->opd_last_used_oid_file->do_lu);
305 d->opd_last_used_oid_file = NULL;
308 if (d->opd_last_used_seq_file != NULL) {
309 lu_object_put(env, &d->opd_last_used_seq_file->do_lu);
310 d->opd_last_used_seq_file = NULL;
314 static int osp_disconnect(struct osp_device *d)
316 struct obd_import *imp;
319 imp = d->opd_obd->u.cli.cl_import;
321 /* Mark import deactivated now, so we don't try to reconnect if any
322 * of the cleanup RPCs fails (e.g. ldlm cancel, etc). We don't
323 * fully deactivate the import, or that would drop all requests. */
324 LASSERT(imp != NULL);
325 spin_lock(&imp->imp_lock);
326 imp->imp_deactive = 1;
327 spin_unlock(&imp->imp_lock);
329 ptlrpc_deactivate_import(imp);
331 /* Some non-replayable imports (MDS's OSCs) are pinged, so just
332 * delete it regardless. (It's safe to delete an import that was
334 (void)ptlrpc_pinger_del_import(imp);
336 rc = ptlrpc_disconnect_import(imp, 0);
337 if (rc == -ETIMEDOUT || rc == -ENOTCONN || rc == -ESHUTDOWN)
340 CERROR("%s: can't disconnect: rc = %d\n",
341 d->opd_obd->obd_name, rc);
343 ptlrpc_invalidate_import(imp);
348 static int osp_shutdown(const struct lu_env *env, struct osp_device *d)
355 rc = osp_disconnect(d);
357 if (!d->opd_connect_mdt) {
358 /* stop precreate thread */
359 osp_precreate_fini(d);
361 /* stop sync thread */
364 /* release last_used file */
365 osp_last_used_fini(env, d);
368 obd_fid_fini(d->opd_obd);
373 static int osp_process_config(const struct lu_env *env,
374 struct lu_device *dev, struct lustre_cfg *lcfg)
376 struct osp_device *d = lu2osp_dev(dev);
377 struct lprocfs_static_vars lvars = { 0 };
382 switch (lcfg->lcfg_command) {
383 case LCFG_PRE_CLEANUP:
384 rc = osp_disconnect(d);
387 lu_dev_del_linkage(dev->ld_site, dev);
388 rc = osp_shutdown(env, d);
391 lprocfs_osp_init_vars(&lvars);
394 rc = class_process_proc_param(PARAM_OSC, lvars.obd_vars,
399 /* class_process_proc_param() haven't found matching
400 * parameter and returned ENOSYS so that layer(s)
401 * below could use that. But OSP is the bottom, so
403 CERROR("%s: unknown param %s\n",
404 (char *)lustre_cfg_string(lcfg, 0),
405 (char *)lustre_cfg_string(lcfg, 1));
410 CERROR("%s: unknown command %u\n",
411 (char *)lustre_cfg_string(lcfg, 0), lcfg->lcfg_command);
419 static int osp_recovery_complete(const struct lu_env *env,
420 struct lu_device *dev)
422 struct osp_device *osp = lu2osp_dev(dev);
426 osp->opd_recovery_completed = 1;
427 if (!osp->opd_connect_mdt)
428 cfs_waitq_signal(&osp->opd_pre_waitq);
432 const struct lu_device_operations osp_lu_ops = {
433 .ldo_object_alloc = osp_object_alloc,
434 .ldo_process_config = osp_process_config,
435 .ldo_recovery_complete = osp_recovery_complete,
439 * provides with statfs from corresponded OST
442 static int osp_statfs(const struct lu_env *env, struct dt_device *dev,
443 struct obd_statfs *sfs)
445 struct osp_device *d = dt2osp_dev(dev);
449 if (unlikely(d->opd_imp_active == 0))
452 /* return recently updated data */
453 *sfs = d->opd_statfs;
456 * layer above osp (usually lod) can use ffree to estimate
457 * how many objects are available for immediate creation
460 spin_lock(&d->opd_pre_lock);
461 LASSERTF(fid_seq(&d->opd_pre_last_created_fid) ==
462 fid_seq(&d->opd_pre_used_fid),
463 "last_created "DFID", next_fid "DFID"\n",
464 PFID(&d->opd_pre_last_created_fid),
465 PFID(&d->opd_pre_used_fid));
466 sfs->os_fprecreated = fid_oid(&d->opd_pre_last_created_fid) -
467 fid_oid(&d->opd_pre_used_fid);
468 sfs->os_fprecreated -= d->opd_pre_reserved;
469 spin_unlock(&d->opd_pre_lock);
471 LASSERT(sfs->os_fprecreated <= OST_MAX_PRECREATE * 2);
473 CDEBUG(D_OTHER, "%s: "LPU64" blocks, "LPU64" free, "LPU64" avail, "
474 LPU64" files, "LPU64" free files\n", d->opd_obd->obd_name,
475 sfs->os_blocks, sfs->os_bfree, sfs->os_bavail,
476 sfs->os_files, sfs->os_ffree);
480 static int osp_sync(const struct lu_env *env, struct dt_device *dev)
485 * XXX: wake up sync thread, command it to start flushing asap?
491 const struct dt_device_operations osp_dt_ops = {
492 .dt_statfs = osp_statfs,
494 .dt_trans_start = osp_trans_start,
495 .dt_trans_stop = osp_trans_stop,
498 static int osp_connect_to_osd(const struct lu_env *env, struct osp_device *m,
501 struct obd_connect_data *data = NULL;
502 struct obd_device *obd;
507 LASSERT(m->opd_storage_exp == NULL);
513 obd = class_name2obd(nextdev);
515 CERROR("%s: can't locate next device: %s\n",
516 m->opd_obd->obd_name, nextdev);
517 GOTO(out, rc = -ENOTCONN);
520 rc = obd_connect(env, &m->opd_storage_exp, obd, &obd->obd_uuid, data,
523 CERROR("%s: cannot connect to next dev %s: rc = %d\n",
524 m->opd_obd->obd_name, nextdev, rc);
528 m->opd_dt_dev.dd_lu_dev.ld_site =
529 m->opd_storage_exp->exp_obd->obd_lu_dev->ld_site;
530 LASSERT(m->opd_dt_dev.dd_lu_dev.ld_site);
531 m->opd_storage = lu2dt_dev(m->opd_storage_exp->exp_obd->obd_lu_dev);
538 static int osp_init0(const struct lu_env *env, struct osp_device *m,
539 struct lu_device_type *ldt, struct lustre_cfg *cfg)
541 struct obd_device *obd;
542 struct obd_import *imp;
544 char *src, *tgt, *mdt, *osdname = NULL;
549 obd = class_name2obd(lustre_cfg_string(cfg, 0));
551 CERROR("Cannot find obd with name %s\n",
552 lustre_cfg_string(cfg, 0));
557 /* There is no record in the MDT configuration for the local disk
558 * device, so we have to extract this from elsewhere in the profile.
559 * The only information we get at setup is from the OSC records:
560 * setup 0:{fsname}-OSTxxxx-osc[-MDTxxxx] 1:lustre-OST0000_UUID 2:NID
561 * Note that 1.8 generated configs are missing the -MDTxxxx part.
562 * We need to reconstruct the name of the underlying OSD from this:
563 * {fsname}-{svname}-osd, for example "lustre-MDT0000-osd". We
564 * also need to determine the OST index from this - will be used
565 * to calculate the offset in shared lov_objids file later */
567 src = lustre_cfg_string(cfg, 0);
571 tgt = strrchr(src, '-');
573 CERROR("%s: invalid target name %s\n",
574 m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
578 if (strncmp(tgt, "-osc", 4) == 0) {
579 /* Old OSC name fsname-OSTXXXX-osc */
580 for (tgt--; tgt > src && *tgt != '-'; tgt--)
583 CERROR("%s: invalid target name %s\n",
584 m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
588 if (strncmp(tgt, "-OST", 4) != 0) {
589 CERROR("%s: invalid target name %s\n",
590 m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
594 idx = simple_strtol(tgt + 4, &mdt, 16);
595 if (mdt[0] != '-' || idx > INT_MAX || idx < 0) {
596 CERROR("%s: invalid OST index in '%s'\n",
597 m->opd_obd->obd_name, src);
604 /* New OSC name fsname-OSTXXXX-osc-MDTXXXX */
605 if (strncmp(tgt, "-MDT", 4) != 0 &&
606 strncmp(tgt, "-OST", 4) != 0) {
607 CERROR("%s: invalid target name %s\n",
608 m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
612 idx = simple_strtol(tgt + 4, &mdt, 16);
613 if (*mdt != '\0' || idx > INT_MAX || idx < 0) {
614 CERROR("%s: invalid OST index in '%s'\n",
615 m->opd_obd->obd_name, src);
619 /* Get MDT index from the name and set it to opd_group,
620 * which will be used by OSP to connect with OST */
622 if (tgt - src <= 12) {
623 CERROR("%s: invalid mdt index retrieve from %s\n",
624 m->opd_obd->obd_name, lustre_cfg_string(cfg, 0));
628 if (strncmp(tgt - 12, "-MDT", 4) == 0)
629 m->opd_connect_mdt = 1;
631 idx = simple_strtol(tgt - 8, &mdt, 16);
632 if (mdt[0] != '-' || idx > INT_MAX || idx < 0) {
633 CERROR("%s: invalid OST index in '%s'\n",
634 m->opd_obd->obd_name, src);
639 idx = tgt - src - 12;
641 /* check the fsname length, and after this everything else will fit */
642 if (idx > MTI_NAME_MAXLEN) {
643 CERROR("%s: fsname too long in '%s'\n",
644 m->opd_obd->obd_name, src);
648 OBD_ALLOC(osdname, MAX_OBD_NAME);
652 memcpy(osdname, src, idx); /* copy just the fsname part */
655 mdt = strstr(mdt, "-MDT");
656 if (mdt == NULL) /* 1.8 configs don't have "-MDT0000" at the end */
657 strcat(osdname, "-MDT0000");
659 strcat(osdname, mdt);
660 strcat(osdname, "-osd");
661 CDEBUG(D_HA, "%s: connect to %s (%s)\n", obd->obd_name, osdname, src);
663 if (m->opd_connect_mdt) {
664 struct client_obd *cli = &m->opd_obd->u.cli;
666 OBD_ALLOC(cli->cl_rpc_lock, sizeof(*cli->cl_rpc_lock));
667 if (!cli->cl_rpc_lock)
669 osp_init_rpc_lock(cli->cl_rpc_lock);
672 m->opd_dt_dev.dd_lu_dev.ld_ops = &osp_lu_ops;
673 m->opd_dt_dev.dd_ops = &osp_dt_ops;
674 obd->obd_lu_dev = &m->opd_dt_dev.dd_lu_dev;
676 rc = osp_connect_to_osd(env, m, osdname);
680 rc = ptlrpcd_addref();
682 GOTO(out_disconnect, rc);
684 rc = client_obd_setup(obd, cfg);
686 CERROR("%s: can't setup obd: %d\n", m->opd_obd->obd_name, rc);
692 if (!m->opd_connect_mdt) {
693 /* Initialize last id from the storage - will be
694 * used in orphan cleanup. */
695 rc = osp_last_used_init(env, m);
698 /* Initialize precreation thread, it handles new
699 * connections as well. */
700 rc = osp_init_precreate(m);
702 GOTO(out_last_used, rc);
704 * Initialize synhronization mechanism taking
705 * care of propogating changes to OST in near
706 * transactional manner.
708 rc = osp_sync_init(env, m);
710 GOTO(out_precreat, rc);
712 rc = obd_fid_init(m->opd_obd, NULL, LUSTRE_SEQ_DATA);
714 CERROR("%s: fid init error: rc = %d\n",
715 m->opd_obd->obd_name, rc);
720 * Initiate connect to OST
722 ll_generate_random_uuid(uuid);
723 class_uuid_unparse(uuid, &m->opd_cluuid);
725 imp = obd->u.cli.cl_import;
727 rc = ptlrpc_init_import(imp);
731 OBD_FREE(osdname, MAX_OBD_NAME);
735 if (!m->opd_connect_mdt)
736 /* stop sync thread */
739 /* stop precreate thread */
740 if (!m->opd_connect_mdt)
741 osp_precreate_fini(m);
743 if (!m->opd_connect_mdt)
744 osp_last_used_fini(env, m);
746 ptlrpc_lprocfs_unregister_obd(obd);
747 lprocfs_obd_cleanup(obd);
748 obd_cleanup_client_import(obd);
750 lprocfs_remove(&m->opd_symlink);
751 client_obd_cleanup(obd);
755 if (m->opd_connect_mdt) {
756 struct client_obd *cli = &m->opd_obd->u.cli;
757 if (cli->cl_rpc_lock != NULL) {
758 OBD_FREE_PTR(cli->cl_rpc_lock);
759 cli->cl_rpc_lock = NULL;
762 obd_disconnect(m->opd_storage_exp);
765 OBD_FREE(osdname, MAX_OBD_NAME);
769 static struct lu_device *osp_device_free(const struct lu_env *env,
770 struct lu_device *lu)
772 struct osp_device *m = lu2osp_dev(lu);
776 if (cfs_atomic_read(&lu->ld_ref) && lu->ld_site) {
777 LIBCFS_DEBUG_MSG_DATA_DECL(msgdata, D_ERROR, NULL);
778 lu_site_print(env, lu->ld_site, &msgdata, lu_cdebug_printer);
780 dt_device_fini(&m->opd_dt_dev);
785 static struct lu_device *osp_device_alloc(const struct lu_env *env,
786 struct lu_device_type *t,
787 struct lustre_cfg *lcfg)
789 struct osp_device *m;
794 l = ERR_PTR(-ENOMEM);
799 dt_device_init(&m->opd_dt_dev, t);
800 rc = osp_init0(env, m, t, lcfg);
802 osp_device_free(env, l);
809 static struct lu_device *osp_device_fini(const struct lu_env *env,
812 struct osp_device *m = lu2osp_dev(d);
813 struct obd_import *imp;
818 if (m->opd_storage_exp)
819 obd_disconnect(m->opd_storage_exp);
821 imp = m->opd_obd->u.cli.cl_import;
823 if (imp->imp_rq_pool) {
824 ptlrpc_free_rq_pool(imp->imp_rq_pool);
825 imp->imp_rq_pool = NULL;
828 obd_cleanup_client_import(m->opd_obd);
831 lprocfs_remove(&m->opd_symlink);
834 ptlrpc_lprocfs_unregister_obd(m->opd_obd);
835 lprocfs_obd_cleanup(m->opd_obd);
837 if (m->opd_connect_mdt) {
838 struct client_obd *cli = &m->opd_obd->u.cli;
839 if (cli->cl_rpc_lock != NULL) {
840 OBD_FREE_PTR(cli->cl_rpc_lock);
841 cli->cl_rpc_lock = NULL;
845 rc = client_obd_cleanup(m->opd_obd);
846 LASSERTF(rc == 0, "error %d\n", rc);
853 static int osp_reconnect(const struct lu_env *env,
854 struct obd_export *exp, struct obd_device *obd,
855 struct obd_uuid *cluuid,
856 struct obd_connect_data *data,
863 * we use exports to track all LOD users
865 static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp,
866 struct obd_device *obd, struct obd_uuid *cluuid,
867 struct obd_connect_data *data, void *localdata)
869 struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev);
870 struct obd_connect_data *ocd;
871 struct obd_import *imp;
872 struct lustre_handle conn;
877 CDEBUG(D_CONFIG, "connect #%d\n", osp->opd_connects);
879 rc = class_connect(&conn, obd, cluuid);
883 *exp = class_conn2export(&conn);
884 /* Why should there ever be more than 1 connect? */
886 LASSERT(osp->opd_connects == 1);
890 imp = osp->opd_obd->u.cli.cl_import;
891 imp->imp_dlm_handle = conn;
893 LASSERT(data != NULL);
894 LASSERT(data->ocd_connect_flags & OBD_CONNECT_INDEX);
895 ocd = &imp->imp_connect_data;
898 imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
900 ocd->ocd_version = LUSTRE_VERSION_CODE;
901 ocd->ocd_index = data->ocd_index;
902 imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
904 rc = ptlrpc_connect_import(imp);
906 CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc);
910 ptlrpc_pinger_add_import(imp);
912 if (osp->opd_connect_mdt && data->ocd_index == 0) {
913 /* set seq controller export for MDC0 if exists */
914 struct seq_server_site *ss;
916 ss = lu_site2seq(osp2lu_dev(osp)->ld_site);
917 ss->ss_control_exp = class_export_get(*exp);
918 ss->ss_server_fld->lsf_control_exp = *exp;
925 * once last export (we don't count self-export) disappeared
926 * osp can be released
928 static int osp_obd_disconnect(struct obd_export *exp)
930 struct obd_device *obd = exp->exp_obd;
931 struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev);
935 /* Only disconnect the underlying layers on the final disconnect. */
936 LASSERT(osp->opd_connects == 1);
939 rc = class_disconnect(exp);
941 CERROR("%s: class disconnect error: rc = %d\n",
946 /* destroy the device */
947 class_manual_cleanup(obd);
953 * lprocfs helpers still use OBD API, let's keep obd_statfs() support
955 static int osp_obd_statfs(const struct lu_env *env, struct obd_export *exp,
956 struct obd_statfs *osfs, __u64 max_age, __u32 flags)
958 struct obd_statfs *msfs;
959 struct ptlrpc_request *req;
960 struct obd_import *imp = NULL;
965 /* Since the request might also come from lprocfs, so we need
966 * sync this with client_disconnect_export Bug15684 */
967 down_read(&exp->exp_obd->u.cli.cl_sem);
968 if (exp->exp_obd->u.cli.cl_import)
969 imp = class_import_get(exp->exp_obd->u.cli.cl_import);
970 up_read(&exp->exp_obd->u.cli.cl_sem);
974 /* We could possibly pass max_age in the request (as an absolute
975 * timestamp or a "seconds.usec ago") so the target can avoid doing
976 * extra calls into the filesystem if that isn't necessary (e.g.
977 * during mount that would help a bit). Having relative timestamps
978 * is not so great if request processing is slow, while absolute
979 * timestamps are not ideal because they need time synchronization. */
980 req = ptlrpc_request_alloc(imp, &RQF_OST_STATFS);
982 class_import_put(imp);
987 rc = ptlrpc_request_pack(req, LUSTRE_OST_VERSION, OST_STATFS);
989 ptlrpc_request_free(req);
992 ptlrpc_request_set_replen(req);
993 req->rq_request_portal = OST_CREATE_PORTAL;
994 ptlrpc_at_set_req_timeout(req);
996 if (flags & OBD_STATFS_NODELAY) {
997 /* procfs requests not want stat in wait for avoid deadlock */
998 req->rq_no_resend = 1;
999 req->rq_no_delay = 1;
1002 rc = ptlrpc_queue_wait(req);
1006 msfs = req_capsule_server_get(&req->rq_pill, &RMF_OBD_STATFS);
1008 GOTO(out, rc = -EPROTO);
1014 ptlrpc_req_finished(req);
1018 static int osp_import_event(struct obd_device *obd, struct obd_import *imp,
1019 enum obd_import_event event)
1021 struct osp_device *d = lu2osp_dev(obd->obd_lu_dev);
1024 case IMP_EVENT_DISCON:
1025 d->opd_got_disconnected = 1;
1026 d->opd_imp_connected = 0;
1027 if (d->opd_connect_mdt)
1029 osp_pre_update_status(d, -ENODEV);
1030 cfs_waitq_signal(&d->opd_pre_waitq);
1031 CDEBUG(D_HA, "got disconnected\n");
1033 case IMP_EVENT_INACTIVE:
1034 d->opd_imp_active = 0;
1035 if (d->opd_connect_mdt)
1037 osp_pre_update_status(d, -ENODEV);
1038 cfs_waitq_signal(&d->opd_pre_waitq);
1039 CDEBUG(D_HA, "got inactive\n");
1041 case IMP_EVENT_ACTIVE:
1042 d->opd_imp_active = 1;
1043 if (d->opd_got_disconnected)
1044 d->opd_new_connection = 1;
1045 d->opd_imp_connected = 1;
1046 d->opd_imp_seen_connected = 1;
1047 if (d->opd_connect_mdt)
1049 cfs_waitq_signal(&d->opd_pre_waitq);
1050 __osp_sync_check_for_work(d);
1051 CDEBUG(D_HA, "got connected\n");
1053 case IMP_EVENT_INVALIDATE:
1054 if (obd->obd_namespace == NULL)
1056 ldlm_namespace_cleanup(obd->obd_namespace, LDLM_FL_LOCAL_ONLY);
1059 case IMP_EVENT_DEACTIVATE:
1060 case IMP_EVENT_ACTIVATE:
1063 CERROR("%s: unsupported import event: %#x\n",
1064 obd->obd_name, event);
1069 static int osp_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
1070 void *karg, void *uarg)
1072 struct obd_device *obd = exp->exp_obd;
1073 struct osp_device *d;
1074 struct obd_ioctl_data *data = karg;
1079 LASSERT(obd->obd_lu_dev);
1080 d = lu2osp_dev(obd->obd_lu_dev);
1081 LASSERT(d->opd_dt_dev.dd_ops == &osp_dt_ops);
1083 if (!cfs_try_module_get(THIS_MODULE)) {
1084 CERROR("%s: can't get module. Is it alive?", obd->obd_name);
1089 case OBD_IOC_CLIENT_RECOVER:
1090 rc = ptlrpc_recover_import(obd->u.cli.cl_import,
1091 data->ioc_inlbuf1, 0);
1095 case IOC_OSC_SET_ACTIVE:
1096 rc = ptlrpc_set_import_active(obd->u.cli.cl_import,
1099 case OBD_IOC_PING_TARGET:
1100 rc = ptlrpc_obd_ping(obd);
1103 CERROR("%s: unrecognized ioctl %#x by %s\n", obd->obd_name,
1104 cmd, cfs_curproc_comm());
1107 cfs_module_put(THIS_MODULE);
1111 static int osp_obd_health_check(const struct lu_env *env,
1112 struct obd_device *obd)
1114 struct osp_device *d = lu2osp_dev(obd->obd_lu_dev);
1119 * 1.8/2.0 behaviour is that OST being connected once at least
1120 * is considired "healthy". and one "healty" OST is enough to
1121 * allow lustre clients to connect to MDS
1124 RETURN(!d->opd_imp_seen_connected);
1127 /* context key constructor/destructor: mdt_key_init, mdt_key_fini */
1128 LU_KEY_INIT_FINI(osp, struct osp_thread_info);
1129 static void osp_key_exit(const struct lu_context *ctx,
1130 struct lu_context_key *key, void *data)
1132 struct osp_thread_info *info = data;
1134 info->osi_attr.la_valid = 0;
1137 struct lu_context_key osp_thread_key = {
1138 .lct_tags = LCT_MD_THREAD,
1139 .lct_init = osp_key_init,
1140 .lct_fini = osp_key_fini,
1141 .lct_exit = osp_key_exit
1144 /* context key constructor/destructor: mdt_txn_key_init, mdt_txn_key_fini */
1145 LU_KEY_INIT_FINI(osp_txn, struct osp_txn_info);
1147 struct lu_context_key osp_txn_key = {
1148 .lct_tags = LCT_OSP_THREAD,
1149 .lct_init = osp_txn_key_init,
1150 .lct_fini = osp_txn_key_fini
1152 LU_TYPE_INIT_FINI(osp, &osp_thread_key, &osp_txn_key);
1154 static struct lu_device_type_operations osp_device_type_ops = {
1155 .ldto_init = osp_type_init,
1156 .ldto_fini = osp_type_fini,
1158 .ldto_start = osp_type_start,
1159 .ldto_stop = osp_type_stop,
1161 .ldto_device_alloc = osp_device_alloc,
1162 .ldto_device_free = osp_device_free,
1164 .ldto_device_fini = osp_device_fini
1167 static struct lu_device_type osp_device_type = {
1168 .ldt_tags = LU_DEVICE_DT,
1169 .ldt_name = LUSTRE_OSP_NAME,
1170 .ldt_ops = &osp_device_type_ops,
1171 .ldt_ctx_tags = LCT_MD_THREAD
1174 static struct obd_ops osp_obd_device_ops = {
1175 .o_owner = THIS_MODULE,
1176 .o_add_conn = client_import_add_conn,
1177 .o_del_conn = client_import_del_conn,
1178 .o_reconnect = osp_reconnect,
1179 .o_connect = osp_obd_connect,
1180 .o_disconnect = osp_obd_disconnect,
1181 .o_health_check = osp_obd_health_check,
1182 .o_import_event = osp_import_event,
1183 .o_iocontrol = osp_iocontrol,
1184 .o_statfs = osp_obd_statfs,
1185 .o_fid_init = client_fid_init,
1186 .o_fid_fini = client_fid_fini,
1189 struct llog_operations osp_mds_ost_orig_logops;
1191 static int __init osp_mod_init(void)
1193 struct lprocfs_static_vars lvars;
1194 cfs_proc_dir_entry_t *osc_proc_dir;
1197 rc = lu_kmem_init(osp_caches);
1201 lprocfs_osp_init_vars(&lvars);
1203 rc = class_register_type(&osp_obd_device_ops, NULL, lvars.module_vars,
1204 LUSTRE_OSP_NAME, &osp_device_type);
1206 /* create "osc" entry in procfs for compatibility purposes */
1208 lu_kmem_fini(osp_caches);
1212 lprocfs_lwp_init_vars(&lvars);
1214 rc = class_register_type(&lwp_obd_device_ops, NULL, lvars.module_vars,
1215 LUSTRE_LWP_NAME, &lwp_device_type);
1217 class_unregister_type(LUSTRE_OSP_NAME);
1218 lu_kmem_fini(osp_caches);
1222 /* Note: add_rec/delcare_add_rec will be only used by catalogs */
1223 osp_mds_ost_orig_logops = llog_osd_ops;
1224 osp_mds_ost_orig_logops.lop_add = llog_cat_add_rec;
1225 osp_mds_ost_orig_logops.lop_declare_add = llog_cat_declare_add_rec;
1227 osc_proc_dir = lprocfs_srch(proc_lustre_root, "osc");
1228 if (osc_proc_dir == NULL) {
1229 osc_proc_dir = lprocfs_register("osc", proc_lustre_root, NULL,
1231 if (IS_ERR(osc_proc_dir))
1232 CERROR("osp: can't create compat entry \"osc\": %d\n",
1233 (int) PTR_ERR(osc_proc_dir));
1238 static void __exit osp_mod_exit(void)
1240 lprocfs_try_remove_proc_entry("osc", proc_lustre_root);
1242 class_unregister_type(LUSTRE_LWP_NAME);
1243 class_unregister_type(LUSTRE_OSP_NAME);
1244 lu_kmem_fini(osp_caches);
1247 MODULE_AUTHOR("Intel, Inc. <http://www.intel.com/>");
1248 MODULE_DESCRIPTION("Lustre OST Proxy Device ("LUSTRE_OSP_NAME")");
1249 MODULE_LICENSE("GPL");
1251 cfs_module(osp, LUSTRE_VERSION_STRING, osp_mod_init, osp_mod_exit);