4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; If not, see
18 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
20 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21 * CA 95054 USA or visit www.sun.com if you need additional information or
27 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Use is subject to license terms.
30 * Copyright (c) 2011, 2012, Intel, Inc.
33 * This file is part of Lustre, http://www.lustre.org/
34 * Lustre is a trademark of Sun Microsystems, Inc.
36 * lustre/osp/osp_dev.c
38 * Lustre OST Proxy Device
40 * Author: Alex Zhuravlev <alexey.zhuravlev@intel.com>
41 * Author: Mikhail Pershin <mike.pershin@intel.com>
45 # define EXPORT_SYMTAB
47 #define DEBUG_SUBSYSTEM S_MDS
49 #include <obd_class.h>
50 #include <lustre_param.h>
52 #include "osp_internal.h"
54 /* Slab for OSP object allocation */
55 cfs_mem_cache_t *osp_object_kmem;
57 static struct lu_kmem_descr osp_caches[] = {
59 .ckd_cache = &osp_object_kmem,
60 .ckd_name = "osp_obj",
61 .ckd_size = sizeof(struct osp_object)
68 struct lu_object *osp_object_alloc(const struct lu_env *env,
69 const struct lu_object_header *hdr,
72 struct lu_object_header *h;
78 OBD_SLAB_ALLOC_PTR_GFP(o, osp_object_kmem, CFS_ALLOC_IO);
80 l = &o->opo_obj.do_lu;
83 lu_object_header_init(h);
84 dt_object_init(&o->opo_obj, h, d);
85 lu_object_add_top(h, l);
87 l->lo_ops = &osp_lu_obj_ops;
95 /* Update opd_last_used_id along with checking for gap in objid sequence */
96 void osp_update_last_id(struct osp_device *d, obd_id objid)
99 * we might have lost precreated objects due to VBR and precreate
100 * orphans, the gap in objid can be calculated properly only here
102 if (objid > le64_to_cpu(d->opd_last_used_id)) {
103 if (objid - le64_to_cpu(d->opd_last_used_id) > 1) {
104 d->opd_gap_start = le64_to_cpu(d->opd_last_used_id) + 1;
105 d->opd_gap_count = objid - d->opd_gap_start;
106 CDEBUG(D_HA, "Gap in objids: %d, start = %llu\n",
107 d->opd_gap_count, d->opd_gap_start);
109 d->opd_last_used_id = cpu_to_le64(objid);
113 static int osp_last_used_init(const struct lu_env *env, struct osp_device *m)
115 struct osp_thread_info *osi = osp_env_info(env);
116 struct dt_object_format dof = { 0 };
122 osi->osi_attr.la_valid = LA_MODE;
123 osi->osi_attr.la_mode = S_IFREG | 0644;
124 lu_local_obj_fid(&osi->osi_fid, MDD_LOV_OBJ_OID);
125 dof.dof_type = DFT_REGULAR;
126 o = dt_find_or_create(env, m->opd_storage, &osi->osi_fid, &dof,
131 rc = dt_attr_get(env, o, &osi->osi_attr, NULL);
135 /* object will be released in device cleanup path */
136 m->opd_last_used_file = o;
138 if (osi->osi_attr.la_size >= sizeof(osi->osi_id) *
139 (m->opd_index + 1)) {
140 osp_objid_buf_prep(osi, m, m->opd_index);
141 rc = dt_record_read(env, o, &osi->osi_lb, &osi->osi_off);
145 /* reset value to 0, just to make sure and change file's size */
148 m->opd_last_used_id = 0;
149 osp_objid_buf_prep(osi, m, m->opd_index);
151 th = dt_trans_create(env, m->opd_storage);
153 GOTO(out, rc = PTR_ERR(th));
155 rc = dt_declare_record_write(env, m->opd_last_used_file,
156 osi->osi_lb.lb_len, osi->osi_off,
159 dt_trans_stop(env, m->opd_storage, th);
163 rc = dt_trans_start_local(env, m->opd_storage, th);
165 dt_trans_stop(env, m->opd_storage, th);
169 rc = dt_record_write(env, m->opd_last_used_file, &osi->osi_lb,
171 dt_trans_stop(env, m->opd_storage, th);
177 CERROR("%s: can't initialize lov_objid: %d\n",
178 m->opd_obd->obd_name, rc);
179 lu_object_put(env, &o->do_lu);
180 m->opd_last_used_file = NULL;
184 static void osp_last_used_fini(const struct lu_env *env, struct osp_device *d)
186 lu_object_put(env, &d->opd_last_used_file->do_lu);
187 d->opd_last_used_file = NULL;
190 static int osp_shutdown(const struct lu_env *env, struct osp_device *d)
192 struct obd_import *imp;
196 /* release last_used file */
197 osp_last_used_fini(env, d);
199 imp = d->opd_obd->u.cli.cl_import;
201 /* Mark import deactivated now, so we don't try to reconnect if any
202 * of the cleanup RPCs fails (e.g. ldlm cancel, etc). We don't
203 * fully deactivate the import, or that would drop all requests. */
204 cfs_spin_lock(&imp->imp_lock);
205 imp->imp_deactive = 1;
206 cfs_spin_unlock(&imp->imp_lock);
208 ptlrpc_deactivate_import(imp);
210 /* Some non-replayable imports (MDS's OSCs) are pinged, so just
211 * delete it regardless. (It's safe to delete an import that was
213 (void)ptlrpc_pinger_del_import(imp);
215 rc = ptlrpc_disconnect_import(imp, 0);
217 CERROR("%s: can't disconnect: rc = %d\n",
218 d->opd_obd->obd_name, rc);
220 ptlrpc_invalidate_import(imp);
222 /* stop precreate thread */
223 osp_precreate_fini(d);
228 static int osp_process_config(const struct lu_env *env,
229 struct lu_device *dev, struct lustre_cfg *lcfg)
231 struct osp_device *d = lu2osp_dev(dev);
232 struct lprocfs_static_vars lvars = { 0 };
237 switch (lcfg->lcfg_command) {
239 lu_dev_del_linkage(dev->ld_site, dev);
240 rc = osp_shutdown(env, d);
243 lprocfs_osp_init_vars(&lvars);
246 rc = class_process_proc_param(PARAM_OSP, lvars.obd_vars,
251 /* class_process_proc_param() haven't found matching
252 * parameter and returned ENOSYS so that layer(s)
253 * below could use that. But OSP is the bottom, so
255 CERROR("%s: unknown param %s\n",
256 (char *)lustre_cfg_string(lcfg, 0),
257 (char *)lustre_cfg_string(lcfg, 1));
262 CERROR("%s: unknown command %u\n",
263 (char *)lustre_cfg_string(lcfg, 0), lcfg->lcfg_command);
271 static int osp_recovery_complete(const struct lu_env *env,
272 struct lu_device *dev)
274 struct osp_device *osp = lu2osp_dev(dev);
278 osp->opd_recovery_completed = 1;
279 cfs_waitq_signal(&osp->opd_pre_waitq);
283 const struct lu_device_operations osp_lu_ops = {
284 .ldo_object_alloc = osp_object_alloc,
285 .ldo_process_config = osp_process_config,
286 .ldo_recovery_complete = osp_recovery_complete,
289 static int osp_sync(const struct lu_env *env, struct dt_device *dev)
294 * XXX: wake up sync thread, command it to start flushing asap?
300 static const struct dt_device_operations osp_dt_ops = {
304 static int osp_connect_to_osd(const struct lu_env *env, struct osp_device *m,
307 struct obd_connect_data *data = NULL;
308 struct obd_device *obd;
313 LASSERT(m->opd_storage_exp == NULL);
319 obd = class_name2obd(nextdev);
321 CERROR("%s: can't locate next device: %s\n",
322 m->opd_obd->obd_name, nextdev);
323 GOTO(out, rc = -ENOTCONN);
326 rc = obd_connect(env, &m->opd_storage_exp, obd, &obd->obd_uuid, data,
329 CERROR("%s: cannot connect to next dev %s: rc = %d\n",
330 m->opd_obd->obd_name, nextdev, rc);
334 m->opd_dt_dev.dd_lu_dev.ld_site =
335 m->opd_storage_exp->exp_obd->obd_lu_dev->ld_site;
336 LASSERT(m->opd_dt_dev.dd_lu_dev.ld_site);
337 m->opd_storage = lu2dt_dev(m->opd_storage_exp->exp_obd->obd_lu_dev);
344 static int osp_init0(const struct lu_env *env, struct osp_device *m,
345 struct lu_device_type *ldt, struct lustre_cfg *cfg)
347 struct lprocfs_static_vars lvars = { 0 };
348 struct proc_dir_entry *osc_proc_dir;
349 struct obd_import *imp;
351 char *src, *ost, *mdt, *osdname = NULL;
356 m->opd_obd = class_name2obd(lustre_cfg_string(cfg, 0));
357 if (m->opd_obd == NULL) {
358 CERROR("Cannot find obd with name %s\n",
359 lustre_cfg_string(cfg, 0));
363 /* There is no record in the MDT configuration for the local disk
364 * device, so we have to extract this from elsewhere in the profile.
365 * The only information we get at setup is from the OSC records:
366 * setup 0:{fsname}-OSTxxxx-osc[-MDTxxxx] 1:lustre-OST0000_UUID 2:NID
367 * Note that 1.8 generated configs are missing the -MDTxxxx part.
368 * We need to reconstruct the name of the underlying OSD from this:
369 * {fsname}-{svname}-osd, for example "lustre-MDT0000-osd". We
370 * also need to determine the OST index from this - will be used
371 * to calculate the offset in shared lov_objids file later */
373 src = lustre_cfg_string(cfg, 0);
377 ost = strstr(src, "-OST");
381 idx = simple_strtol(ost + 4, &mdt, 16);
382 if (mdt[0] != '-' || idx > INT_MAX || idx < 0) {
383 CERROR("%s: invalid OST index in '%s'\n",
384 m->opd_obd->obd_name, src);
385 GOTO(out_fini, rc = -EINVAL);
390 /* check the fsname length, and after this everything else will fit */
391 if (idx > MTI_NAME_MAXLEN) {
392 CERROR("%s: fsname too long in '%s'\n",
393 m->opd_obd->obd_name, src);
394 GOTO(out_fini, rc = -EINVAL);
397 OBD_ALLOC(osdname, MAX_OBD_NAME);
399 GOTO(out_fini, rc = -ENOMEM);
401 memcpy(osdname, src, idx); /* copy just the fsname part */
404 mdt = strstr(mdt, "-MDT");
405 if (mdt == NULL) /* 1.8 configs don't have "-MDT0000" at the end */
406 strcat(osdname, "-MDT0000");
408 strcat(osdname, mdt);
409 strcat(osdname, "-osd");
410 CDEBUG(D_HA, "%s: connect to %s (%s)\n",
411 m->opd_obd->obd_name, osdname, src);
413 m->opd_dt_dev.dd_lu_dev.ld_ops = &osp_lu_ops;
414 m->opd_dt_dev.dd_ops = &osp_dt_ops;
415 m->opd_obd->obd_lu_dev = &m->opd_dt_dev.dd_lu_dev;
417 rc = osp_connect_to_osd(env, m, osdname);
421 rc = ptlrpcd_addref();
423 GOTO(out_disconnect, rc);
425 rc = client_obd_setup(m->opd_obd, cfg);
427 CERROR("%s: can't setup obd: %d\n", m->opd_obd->obd_name, rc);
431 lprocfs_osp_init_vars(&lvars);
432 if (lprocfs_obd_setup(m->opd_obd, lvars.obd_vars) == 0)
433 ptlrpc_lprocfs_register_obd(m->opd_obd);
435 /* for compatibility we link old procfs's OSC entries to osp ones */
436 osc_proc_dir = lprocfs_srch(proc_lustre_root, "osc");
438 cfs_proc_dir_entry_t *symlink = NULL;
441 OBD_ALLOC(name, strlen(m->opd_obd->obd_name) + 1);
443 GOTO(out, rc = -ENOMEM);
445 strcpy(name, m->opd_obd->obd_name);
446 if (strstr(name, "osc"))
447 symlink = lprocfs_add_symlink(name, osc_proc_dir,
449 m->opd_obd->obd_name);
450 OBD_FREE(name, strlen(m->opd_obd->obd_name) + 1);
451 m->opd_symlink = symlink;
455 * Initialize last id from the storage - will be used in orphan cleanup
457 rc = osp_last_used_init(env, m);
462 * Initialize precreation thread, it handles new connections as well
464 rc = osp_init_precreate(m);
466 GOTO(out_last_used, rc);
469 * Initiate connect to OST
471 ll_generate_random_uuid(uuid);
472 class_uuid_unparse(uuid, &m->opd_cluuid);
474 imp = m->opd_obd->u.cli.cl_import;
476 rc = ptlrpc_init_import(imp);
480 OBD_FREE(osdname, MAX_OBD_NAME);
484 /* stop precreate thread */
485 osp_precreate_fini(m);
487 osp_last_used_fini(env, m);
489 ptlrpc_lprocfs_unregister_obd(m->opd_obd);
490 lprocfs_obd_cleanup(m->opd_obd);
491 class_destroy_import(m->opd_obd->u.cli.cl_import);
492 client_obd_cleanup(m->opd_obd);
496 obd_disconnect(m->opd_storage_exp);
499 OBD_FREE(osdname, MAX_OBD_NAME);
503 static struct lu_device *osp_device_free(const struct lu_env *env,
504 struct lu_device *lu)
506 struct osp_device *m = lu2osp_dev(lu);
510 dt_device_fini(&m->opd_dt_dev);
515 static struct lu_device *osp_device_alloc(const struct lu_env *env,
516 struct lu_device_type *t,
517 struct lustre_cfg *lcfg)
519 struct osp_device *m;
524 l = ERR_PTR(-ENOMEM);
529 dt_device_init(&m->opd_dt_dev, t);
530 rc = osp_init0(env, m, t, lcfg);
532 osp_device_free(env, l);
539 static struct lu_device *osp_device_fini(const struct lu_env *env,
542 struct osp_device *m = lu2osp_dev(d);
543 struct obd_import *imp;
548 LASSERT(m->opd_storage_exp);
549 obd_disconnect(m->opd_storage_exp);
551 imp = m->opd_obd->u.cli.cl_import;
553 if (imp->imp_rq_pool) {
554 ptlrpc_free_rq_pool(imp->imp_rq_pool);
555 imp->imp_rq_pool = NULL;
558 obd_cleanup_client_import(m->opd_obd);
561 lprocfs_remove(&m->opd_symlink);
564 ptlrpc_lprocfs_unregister_obd(m->opd_obd);
565 lprocfs_obd_cleanup(m->opd_obd);
567 rc = client_obd_cleanup(m->opd_obd);
568 LASSERTF(rc == 0, "error %d\n", rc);
575 static int osp_reconnect(const struct lu_env *env,
576 struct obd_export *exp, struct obd_device *obd,
577 struct obd_uuid *cluuid,
578 struct obd_connect_data *data,
585 * we use exports to track all LOD users
587 static int osp_obd_connect(const struct lu_env *env, struct obd_export **exp,
588 struct obd_device *obd, struct obd_uuid *cluuid,
589 struct obd_connect_data *data, void *localdata)
591 struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev);
592 struct obd_connect_data *ocd;
593 struct obd_import *imp;
594 struct lustre_handle conn;
599 CDEBUG(D_CONFIG, "connect #%d\n", osp->opd_connects);
601 rc = class_connect(&conn, obd, cluuid);
605 *exp = class_conn2export(&conn);
607 /* Why should there ever be more than 1 connect? */
609 LASSERT(osp->opd_connects == 1);
611 imp = osp->opd_obd->u.cli.cl_import;
612 imp->imp_dlm_handle = conn;
614 ocd = &imp->imp_connect_data;
615 ocd->ocd_connect_flags = OBD_CONNECT_AT |
618 #ifdef HAVE_LRU_RESIZE_SUPPORT
619 OBD_CONNECT_LRU_RESIZE |
622 OBD_CONNECT_OSS_CAPA |
623 OBD_CONNECT_REQPORTAL |
624 OBD_CONNECT_SKIP_ORPHAN |
626 ocd->ocd_version = LUSTRE_VERSION_CODE;
627 LASSERT(data->ocd_connect_flags & OBD_CONNECT_INDEX);
628 ocd->ocd_index = data->ocd_index;
629 imp->imp_connect_flags_orig = ocd->ocd_connect_flags;
631 rc = ptlrpc_connect_import(imp);
633 CERROR("%s: can't connect obd: rc = %d\n", obd->obd_name, rc);
637 ptlrpc_pinger_add_import(imp);
644 * once last export (we don't count self-export) disappeared
645 * osp can be released
647 static int osp_obd_disconnect(struct obd_export *exp)
649 struct obd_device *obd = exp->exp_obd;
650 struct osp_device *osp = lu2osp_dev(obd->obd_lu_dev);
655 /* Only disconnect the underlying layers on the final disconnect. */
656 LASSERT(osp->opd_connects == 1);
659 rc = class_disconnect(exp);
661 /* destroy the device */
663 class_manual_cleanup(obd);
668 /* context key constructor/destructor: mdt_key_init, mdt_key_fini */
669 LU_KEY_INIT_FINI(osp, struct osp_thread_info);
670 static void osp_key_exit(const struct lu_context *ctx,
671 struct lu_context_key *key, void *data)
673 struct osp_thread_info *info = data;
675 info->osi_attr.la_valid = 0;
678 struct lu_context_key osp_thread_key = {
679 .lct_tags = LCT_MD_THREAD,
680 .lct_init = osp_key_init,
681 .lct_fini = osp_key_fini,
682 .lct_exit = osp_key_exit
685 /* context key constructor/destructor: mdt_txn_key_init, mdt_txn_key_fini */
686 LU_KEY_INIT_FINI(osp_txn, struct osp_txn_info);
688 struct lu_context_key osp_txn_key = {
689 .lct_tags = LCT_OSP_THREAD,
690 .lct_init = osp_txn_key_init,
691 .lct_fini = osp_txn_key_fini
693 LU_TYPE_INIT_FINI(osp, &osp_thread_key, &osp_txn_key);
695 static struct lu_device_type_operations osp_device_type_ops = {
696 .ldto_init = osp_type_init,
697 .ldto_fini = osp_type_fini,
699 .ldto_start = osp_type_start,
700 .ldto_stop = osp_type_stop,
702 .ldto_device_alloc = osp_device_alloc,
703 .ldto_device_free = osp_device_free,
705 .ldto_device_fini = osp_device_fini
708 static struct lu_device_type osp_device_type = {
709 .ldt_tags = LU_DEVICE_DT,
710 .ldt_name = LUSTRE_OSP_NAME,
711 .ldt_ops = &osp_device_type_ops,
712 .ldt_ctx_tags = LCT_MD_THREAD
715 static struct obd_ops osp_obd_device_ops = {
716 .o_owner = THIS_MODULE,
717 .o_add_conn = client_import_add_conn,
718 .o_del_conn = client_import_del_conn,
719 .o_reconnect = osp_reconnect,
720 .o_connect = osp_obd_connect,
721 .o_disconnect = osp_obd_disconnect,
724 static int __init osp_mod_init(void)
726 struct lprocfs_static_vars lvars;
727 cfs_proc_dir_entry_t *osc_proc_dir;
730 rc = lu_kmem_init(osp_caches);
734 lprocfs_osp_init_vars(&lvars);
736 rc = class_register_type(&osp_obd_device_ops, NULL, lvars.module_vars,
737 LUSTRE_OSP_NAME, &osp_device_type);
739 /* create "osc" entry in procfs for compatibility purposes */
741 lu_kmem_fini(osp_caches);
745 osc_proc_dir = lprocfs_srch(proc_lustre_root, "osc");
746 if (osc_proc_dir == NULL) {
747 osc_proc_dir = lprocfs_register("osc", proc_lustre_root, NULL,
749 if (IS_ERR(osc_proc_dir))
750 CERROR("osp: can't create compat entry \"osc\": %d\n",
751 (int) PTR_ERR(osc_proc_dir));
756 static void __exit osp_mod_exit(void)
758 lprocfs_try_remove_proc_entry("osc", proc_lustre_root);
760 class_unregister_type(LUSTRE_OSP_NAME);
761 lu_kmem_fini(osp_caches);
764 MODULE_AUTHOR("Intel, Inc. <http://www.intel.com/>");
765 MODULE_DESCRIPTION("Lustre OST Proxy Device ("LUSTRE_OSP_NAME")");
766 MODULE_LICENSE("GPL");
768 cfs_module(osp, LUSTRE_VERSION_STRING, osp_mod_init, osp_mod_exit);