1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
5 * Lustre Metadata Server (mds) handling of striped file data
7 * Copyright (C) 2001-2006 Cluster File Systems, Inc.
8 * Author: Peter Braam <braam@clusterfs.com>
9 * wangdi <wangdi@clusterfs.com>
11 * This file is part of the Lustre file system, http://www.lustre.org
12 * Lustre is a trademark of Cluster File Systems, Inc.
14 * You may have signed or agreed to another license before downloading
15 * this software. If so, you are bound by the terms and conditions
16 * of that agreement, and the following does not apply to you. See the
17 * LICENSE file included with this distribution for more information.
19 * If you did not agree to a different license, then this copy of Lustre
20 * is open source software; you can redistribute it and/or modify it
21 * under the terms of version 2 of the GNU General Public License as
22 * published by the Free Software Foundation.
24 * In either case, Lustre is distributed in the hope that it will be
25 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty
26 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
27 * license text for more details.
30 # define EXPORT_SYMTAB
32 #define DEBUG_SUBSYSTEM S_MDS
34 #include <linux/module.h>
36 #include <obd_class.h>
37 #include <lustre_ver.h>
38 #include <obd_support.h>
40 #include <lprocfs_status.h>
41 #include <lustre_mds.h>
42 #include <lustre_fid.h>
43 #include <lustre/lustre_idl.h>
45 #include "mdd_internal.h"
47 static int mdd_notify(struct obd_device *host, struct obd_device *watched,
48 enum obd_notify_event ev, void *owner)
50 struct mdd_device *mdd = owner;
54 LASSERT(owner != NULL);
57 case OBD_NOTIFY_ACTIVE:
59 case OBD_NOTIFY_SYNC_NONBLOCK:
60 rc = md_do_upcall(NULL, &mdd->mdd_md_dev, MD_LOV_SYNC);
62 case OBD_NOTIFY_CONFIG:
63 rc = md_do_upcall(NULL, &mdd->mdd_md_dev, MD_LOV_CONFIG);
66 CDEBUG(D_INFO, "Unhandled notification %#x\n", ev);
72 /* The obd is created for handling data stack for mdd */
73 int mdd_init_obd(const struct lu_env *env, struct mdd_device *mdd,
74 struct lustre_cfg *cfg)
76 char *dev = lustre_cfg_string(cfg, 0);
77 int rc, name_size, uuid_size;
80 struct lustre_cfg_bufs *bufs;
81 struct lustre_cfg *lcfg;
82 struct obd_device *obd;
85 mds_id = mdd2lu_dev(mdd)->ld_site->ls_node_id;
86 name_size = strlen(MDD_OBD_NAME) + 35;
87 uuid_size = strlen(MDD_OBD_UUID) + 35;
89 OBD_ALLOC(name, name_size);
90 OBD_ALLOC(uuid, uuid_size);
91 if (name == NULL || uuid == NULL)
92 GOTO(cleanup_mem, rc = -ENOMEM);
96 GOTO(cleanup_mem, rc = -ENOMEM);
98 snprintf(name, strlen(MDD_OBD_NAME) + 35, "%s-%s-%d",
99 MDD_OBD_NAME, dev, mds_id);
101 snprintf(uuid, strlen(MDD_OBD_UUID) + 35, "%s-%s-%d",
102 MDD_OBD_UUID, dev, mds_id);
104 lustre_cfg_bufs_reset(bufs, name);
105 lustre_cfg_bufs_set_string(bufs, 1, MDD_OBD_TYPE);
106 lustre_cfg_bufs_set_string(bufs, 2, uuid);
107 lustre_cfg_bufs_set_string(bufs, 3, (char*)dev/* MDD_OBD_PROFILE */);
108 lustre_cfg_bufs_set_string(bufs, 4, (char*)dev);
110 lcfg = lustre_cfg_new(LCFG_ATTACH, bufs);
113 GOTO(cleanup_mem, rc = -ENOMEM);
115 rc = class_attach(lcfg);
117 GOTO(lcfg_cleanup, rc);
119 obd = class_name2obd(name);
121 CERROR("Can not find obd %s\n", MDD_OBD_NAME);
125 obd->obd_recovering = 1;
126 obd->u.mds.mds_id = mds_id;
127 rc = class_setup(obd, lcfg);
129 GOTO(class_detach, rc);
132 * Add here for obd notify mechanism, when adding a new ost, the mds
133 * will notify this mdd.
135 obd->obd_upcall.onu_upcall = mdd_notify;
136 obd->obd_upcall.onu_owner = mdd;
137 mdd->mdd_obd_dev = obd;
142 class_detach(obd, lcfg);
144 lustre_cfg_free(lcfg);
147 OBD_FREE(name, name_size);
149 OBD_FREE(uuid, uuid_size);
153 int mdd_fini_obd(const struct lu_env *env, struct mdd_device *mdd,
154 struct lustre_cfg *lcfg)
156 struct obd_device *obd;
160 obd = mdd2obd_dev(mdd);
163 rc = class_cleanup(obd, lcfg);
165 GOTO(lcfg_cleanup, rc);
167 obd->obd_upcall.onu_upcall = NULL;
168 obd->obd_upcall.onu_owner = NULL;
169 rc = class_detach(obd, lcfg);
171 GOTO(lcfg_cleanup, rc);
172 mdd->mdd_obd_dev = NULL;
179 int mdd_get_md(const struct lu_env *env, struct mdd_object *obj,
180 void *md, int *md_size, const char *name)
185 rc = mdo_xattr_get(env, obj, mdd_buf_get(env, md, *md_size), name,
186 mdd_object_capa(env, obj));
188 * XXX: Handling of -ENODATA, the right way is to have ->do_md_get()
189 * exported by dt layer.
191 if (rc == 0 || rc == -ENODATA) {
195 CERROR("Error %d reading eadata \n", rc);
197 /* XXX: Convert lov EA but fixed after verification test. */
204 int mdd_get_md_locked(const struct lu_env *env, struct mdd_object *obj,
205 void *md, int *md_size, const char *name)
208 mdd_read_lock(env, obj);
209 rc = mdd_get_md(env, obj, md, md_size, name);
210 mdd_read_unlock(env, obj);
214 static int mdd_lov_set_stripe_md(const struct lu_env *env,
215 struct mdd_object *obj, struct lu_buf *buf,
216 struct thandle *handle)
218 struct mdd_device *mdd = mdo2mdd(&obj->mod_obj);
219 struct obd_device *obd = mdd2obd_dev(mdd);
220 struct obd_export *lov_exp = obd->u.mds.mds_osc_exp;
221 struct lov_stripe_md *lsm = NULL;
225 LASSERT(S_ISDIR(mdd_object_type(obj)) || S_ISREG(mdd_object_type(obj)));
226 rc = obd_iocontrol(OBD_IOC_LOV_SETSTRIPE, lov_exp, 0,
230 obd_free_memmd(lov_exp, &lsm);
232 rc = mdd_xattr_set_txn(env, obj, buf, MDS_LOV_MD_NAME, 0, handle);
234 CDEBUG(D_INFO, "set lov ea of "DFID" rc %d \n", PFID(mdo2fid(obj)), rc);
239 * Permission check is done before call it,
240 * no need check again.
242 static int mdd_lov_set_dir_md(const struct lu_env *env,
243 struct mdd_object *obj, struct lu_buf *buf,
244 struct thandle *handle)
246 struct lov_user_md *lum = NULL;
250 LASSERT(S_ISDIR(mdd_object_type(obj)));
251 lum = (struct lov_user_md*)buf->lb_buf;
253 /* if { size, offset, count } = { 0, -1, 0 } (i.e. all default
254 * values specified) then delete default striping from dir. */
255 if ((lum->lmm_stripe_size == 0 && lum->lmm_stripe_count == 0 &&
256 lum->lmm_stripe_offset == (typeof(lum->lmm_stripe_offset))(-1)) ||
257 /* lmm_stripe_size == -1 is deprecated in 1.4.6 */
258 lum->lmm_stripe_size == (typeof(lum->lmm_stripe_size))(-1)){
259 rc = mdd_xattr_set_txn(env, obj, &LU_BUF_NULL,
260 MDS_LOV_MD_NAME, 0, handle);
263 CDEBUG(D_INFO, "delete lov ea of "DFID" rc %d \n",
264 PFID(mdo2fid(obj)), rc);
266 rc = mdd_lov_set_stripe_md(env, obj, buf, handle);
271 int mdd_lsm_sanity_check(const struct lu_env *env, struct mdd_object *obj)
273 struct lu_attr *tmp_la = &mdd_env_info(env)->mti_la;
274 struct md_ucred *uc = md_ucred(env);
278 rc = mdd_la_get(env, obj, tmp_la, BYPASS_CAPA);
282 if ((uc->mu_fsuid != tmp_la->la_uid) && !mdd_capable(uc, CAP_FOWNER))
283 rc = mdd_permission_internal_locked(env, obj, tmp_la,
289 int mdd_lov_set_md(const struct lu_env *env, struct mdd_object *pobj,
290 struct mdd_object *child, struct lov_mds_md *lmmp,
291 int lmm_size, struct thandle *handle, int set_stripe)
298 buf = mdd_buf_get(env, lmmp, lmm_size);
299 mode = mdd_object_type(child);
300 if (S_ISREG(mode) && lmm_size > 0) {
302 rc = mdd_lov_set_stripe_md(env, child, buf, handle);
304 rc = mdd_xattr_set_txn(env, child, buf,
305 MDS_LOV_MD_NAME, 0, handle);
307 } else if (S_ISDIR(mode)) {
308 if (lmmp == NULL && lmm_size == 0) {
309 struct mdd_device *mdd = mdd_obj2mdd_dev(child);
310 struct lov_mds_md *lmm = mdd_max_lmm_get(env, mdd);
311 int size = sizeof(*lmm);
313 /* Get parent dir stripe and set */
315 rc = mdd_get_md_locked(env, pobj, lmm, &size,
318 buf = mdd_buf_get(env, lmm, size);
319 rc = mdd_xattr_set_txn(env, child, buf,
320 MDS_LOV_MD_NAME, 0, handle);
322 CERROR("error on copy stripe info: rc "
326 LASSERT(lmmp != NULL && lmm_size > 0);
327 rc = mdd_lov_set_dir_md(env, child, buf, handle);
330 CDEBUG(D_INFO, "Set lov md %p size %d for fid "DFID" rc %d\n",
331 lmmp, lmm_size, PFID(mdo2fid(child)), rc);
336 * XXX: this is for create lsm object id, which should identify the lsm object
337 * unique in the whole mds, as I see. But it seems, we still not need it
338 * now. Right? So just borrow the ll_fid_build_ino().
340 static obd_id mdd_lov_create_id(const struct lu_fid *fid)
342 return fid_flatten(fid);
345 void mdd_lov_objid_update(struct mdd_device *mdd, struct lov_mds_md *lmm)
347 mds_lov_update_objids(mdd->mdd_obd_dev, lmm);
350 void mdd_lov_create_finish(const struct lu_env *env, struct mdd_device *mdd,
351 struct lov_mds_md *lmm, int lmm_size,
352 const struct md_op_spec *spec)
354 if (lmm && !spec->u.sp_ea.no_lov_create)
355 OBD_FREE(lmm, lmm_size);
358 int mdd_lov_create(const struct lu_env *env, struct mdd_device *mdd,
359 struct mdd_object *parent, struct mdd_object *child,
360 struct lov_mds_md **lmm, int *lmm_size,
361 const struct md_op_spec *spec, struct lu_attr *la)
363 struct obd_device *obd = mdd2obd_dev(mdd);
364 struct obd_export *lov_exp = obd->u.mds.mds_osc_exp;
366 struct lov_stripe_md *lsm = NULL;
367 const void *eadata = spec->u.sp_ea.eadata;
368 __u32 create_flags = spec->sp_cr_flags;
369 struct obd_trans_info *oti = &mdd_env_info(env)->mti_oti;
373 if (!md_should_create(create_flags))
378 /* replay case, has objects already, only get lov from eadata */
379 if (spec->u.sp_ea.no_lov_create != 0) {
380 *lmm = (struct lov_mds_md *)spec->u.sp_ea.eadata;
381 *lmm_size = spec->u.sp_ea.eadatalen;
385 if (OBD_FAIL_CHECK_ONCE(OBD_FAIL_MDS_ALLOC_OBDO))
386 GOTO(out_ids, rc = -ENOMEM);
388 LASSERT(lov_exp != NULL);
389 oa = &mdd_env_info(env)->mti_oa;
391 oa->o_uid = 0; /* must have 0 uid / gid on OST */
393 oa->o_gr = FILTER_GROUP_MDS0 + mdd2lu_dev(mdd)->ld_site->ls_node_id;
394 oa->o_mode = S_IFREG | 0600;
395 oa->o_id = mdd_lov_create_id(mdd_object_fid(child));
396 oa->o_valid = OBD_MD_FLID | OBD_MD_FLTYPE | OBD_MD_FLFLAGS |
397 OBD_MD_FLMODE | OBD_MD_FLUID | OBD_MD_FLGID | OBD_MD_FLGROUP;
400 if (!(create_flags & MDS_OPEN_HAS_OBJS)) {
401 if (create_flags & MDS_OPEN_HAS_EA) {
402 LASSERT(eadata != NULL);
403 rc = obd_iocontrol(OBD_IOC_LOV_SETSTRIPE, lov_exp,
404 0, &lsm, (void*)eadata);
407 lsm->lsm_object_id = oa->o_id;
408 lsm->lsm_object_gr = oa->o_gr;
409 } else if (parent != NULL) {
410 /* get lov ea from parent and set to lov */
411 struct lov_mds_md *_lmm;
414 _lmm_size = mdd_lov_mdsize(env, mdd);
415 _lmm = mdd_max_lmm_get(env, mdd);
418 GOTO(out_oti, rc = -ENOMEM);
420 rc = mdd_get_md_locked(env, parent, _lmm,
424 rc = obd_iocontrol(OBD_IOC_LOV_SETSTRIPE,
425 lov_exp, 0, &lsm, _lmm);
430 rc = obd_create(lov_exp, oa, &lsm, oti);
433 CERROR("Create error for "DFID": %d\n",
434 PFID(mdo2fid(child)), rc);
439 LASSERT(lsm->lsm_object_gr >= FILTER_GROUP_MDS0);
441 LASSERT(eadata != NULL);
442 rc = obd_iocontrol(OBD_IOC_LOV_SETEA, lov_exp, 0, &lsm,
446 lsm->lsm_object_id = oa->o_id;
447 lsm->lsm_object_gr = oa->o_gr;
451 * Sometimes, we may truncate some object(without lsm) then open it
452 * (with write flags), so creating lsm above. The Nonzero(truncated)
453 * size should tell ost, since size attr is in charge by OST.
455 if (la->la_size && la->la_valid & LA_SIZE) {
456 struct obd_info *oinfo = &mdd_env_info(env)->mti_oi;
458 memset(oinfo, 0, sizeof(*oinfo));
460 oa->o_size = la->la_size;
462 /* When setting attr to ost, FLBKSZ is not needed. */
463 oa->o_valid &= ~OBD_MD_FLBLKSZ;
464 obdo_from_la(oa, la, OBD_MD_FLTYPE | OBD_MD_FLATIME |
465 OBD_MD_FLMTIME | OBD_MD_FLCTIME | OBD_MD_FLSIZE);
468 * XXX: Pack lustre id to OST, in OST, it will be packed by
469 * filter_fid, but can not see what is the usages. So just pack
470 * o_seq o_ver here, maybe fix it after this cycle.
472 oa->o_fid = fid_seq(mdd_object_fid(child));
473 oa->o_generation = fid_oid(mdd_object_fid(child));
474 oa->o_valid |= OBD_MD_FLFID | OBD_MD_FLGENER;
477 oinfo->oi_capa = mdo_capa_get(env, child, NULL,
478 CAPA_OPC_MDS_DEFAULT);
479 if (IS_ERR(oinfo->oi_capa))
480 oinfo->oi_capa = NULL;
482 rc = obd_setattr(lov_exp, oinfo, oti);
483 capa_put(oinfo->oi_capa);
485 CERROR("Error setting attrs for "DFID": rc %d\n",
486 PFID(mdo2fid(child)), rc);
488 CERROR("obd_setattr for "DFID" rc %d\n",
489 PFID(mdo2fid(child)), rc);
496 /* blksize should be changed after create data object */
497 la->la_valid |= LA_BLKSIZE;
498 la->la_blksize = oa->o_blksize;
500 rc = obd_packmd(lov_exp, lmm, lsm);
502 CERROR("Cannot pack lsm, err = %d\n", rc);
509 oti_free_cookies(oti);
512 obd_free_memmd(lov_exp, &lsm);
518 * used when destroying orphans and from mds_reint_unlink() when MDS wants to
519 * destroy objects on OSS.
522 int mdd_lovobj_unlink(const struct lu_env *env, struct mdd_device *mdd,
523 struct mdd_object *obj, struct lu_attr *la,
524 struct lov_mds_md *lmm, int lmm_size,
525 struct llog_cookie *logcookies,
528 struct obd_device *obd = mdd2obd_dev(mdd);
529 struct obd_export *lov_exp = obd->u.mds.mds_osc_exp;
530 struct lov_stripe_md *lsm = NULL;
531 struct obd_trans_info *oti = &mdd_env_info(env)->mti_oti;
532 struct obdo *oa = &mdd_env_info(env)->mti_oa;
539 rc = obd_unpackmd(lov_exp, &lsm, lmm, lmm_size);
541 CERROR("Error unpack md %p\n", lmm);
544 LASSERT(rc >= sizeof(*lsm));
548 oa->o_id = lsm->lsm_object_id;
549 oa->o_gr = FILTER_GROUP_MDS0 + mdd2lu_dev(mdd)->ld_site->ls_node_id;
550 oa->o_mode = la->la_mode & S_IFMT;
551 oa->o_valid = OBD_MD_FLID | OBD_MD_FLTYPE | OBD_MD_FLGROUP;
554 if (log_unlink && logcookies) {
555 oa->o_valid |= OBD_MD_FLCOOKIE;
556 oti->oti_logcookies = logcookies;
559 CDEBUG(D_INFO, "destroying OSS object %d/%d\n",
560 (int)oa->o_id, (int)oa->o_gr);
562 rc = obd_destroy(lov_exp, oa, lsm, oti, NULL);
564 obd_free_memmd(lov_exp, &lsm);
570 * called with obj not locked.
572 int mdd_lov_destroy(const struct lu_env *env, struct mdd_device *mdd,
573 struct mdd_object *obj, struct lu_attr *la)
575 struct md_attr *ma = &mdd_env_info(env)->mti_ma;
579 if (unlikely(la->la_nlink != 0)) {
580 CWARN("Attempt to destroy OSS object when nlink == %d\n",
585 ma->ma_lmm_size = mdd_lov_mdsize(env, mdd);
586 ma->ma_lmm = mdd_max_lmm_get(env, mdd);
587 ma->ma_cookie_size = mdd_lov_cookiesize(env, mdd);
588 ma->ma_cookie = mdd_max_cookie_get(env, mdd);
589 if (ma->ma_lmm == NULL || ma->ma_cookie == NULL)
590 RETURN(rc = -ENOMEM);
593 rc = mdd_get_md_locked(env, obj, ma->ma_lmm, &ma->ma_lmm_size,
596 CWARN("Get lov ea failed for "DFID"\n", PFID(mdo2fid(obj)));
599 ma->ma_valid = MA_LOV;
601 rc = mdd_unlink_log(env, mdd, obj, ma);
603 CWARN("mds unlink log for "DFID" failed: %d\n",
604 PFID(mdo2fid(obj)), rc);
607 if (ma->ma_valid | MA_COOKIE)
608 rc = mdd_lovobj_unlink(env, mdd, obj, la,
609 ma->ma_lmm, ma->ma_lmm_size,
615 int mdd_unlink_log(const struct lu_env *env, struct mdd_device *mdd,
616 struct mdd_object *mdd_cobj, struct md_attr *ma)
618 struct obd_device *obd = mdd2obd_dev(mdd);
620 LASSERT(ma->ma_valid & MA_LOV);
622 if ((ma->ma_cookie_size > 0) &&
623 (mds_log_op_unlink(obd, ma->ma_lmm, ma->ma_lmm_size,
624 ma->ma_cookie, ma->ma_cookie_size) > 0)) {
625 ma->ma_valid |= MA_COOKIE;
630 int mdd_setattr_log(const struct lu_env *env, struct mdd_device *mdd,
631 const struct md_attr *ma,
632 struct lov_mds_md *lmm, int lmm_size,
633 struct llog_cookie *logcookies, int cookies_size)
635 struct obd_device *obd = mdd2obd_dev(mdd);
637 /* journal chown/chgrp in llog, just like unlink */
639 CDEBUG(D_INFO, "setattr llog for uid/gid=%lu/%lu\n",
640 (unsigned long)ma->ma_attr.la_uid,
641 (unsigned long)ma->ma_attr.la_gid);
642 return mds_log_op_setattr(obd, ma->ma_attr.la_uid,
643 ma->ma_attr.la_gid, lmm,
644 lmm_size, logcookies,
650 int mdd_lov_setattr_async(const struct lu_env *env, struct mdd_object *obj,
651 struct lov_mds_md *lmm, int lmm_size,
652 struct llog_cookie *logcookies)
654 struct mdd_device *mdd = mdo2mdd(&obj->mod_obj);
655 struct obd_device *obd = mdd2obd_dev(mdd);
656 struct lu_attr *tmp_la = &mdd_env_info(env)->mti_la;
657 const struct lu_fid *fid = mdd_object_fid(obj);
662 mdd_read_lock(env, obj);
663 rc = mdo_attr_get(env, obj, tmp_la, mdd_object_capa(env, obj));
664 mdd_read_unlock(env, obj);
668 oc = mdo_capa_get(env, obj, NULL, CAPA_OPC_MDS_DEFAULT);
672 rc = mds_osc_setattr_async(obd, tmp_la->la_uid, tmp_la->la_gid, lmm,
673 lmm_size, logcookies, fid_seq(fid),