1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
5 * Lustre Metadata Server (mds) handling of striped file data
7 * Copyright (C) 2001-2006 Cluster File Systems, Inc.
8 * Author: Peter Braam <braam@clusterfs.com>
9 * wangdi <wangdi@clusterfs.com>
11 * This file is part of the Lustre file system, http://www.lustre.org
12 * Lustre is a trademark of Cluster File Systems, Inc.
14 * You may have signed or agreed to another license before downloading
15 * this software. If so, you are bound by the terms and conditions
16 * of that agreement, and the following does not apply to you. See the
17 * LICENSE file included with this distribution for more information.
19 * If you did not agree to a different license, then this copy of Lustre
20 * is open source software; you can redistribute it and/or modify it
21 * under the terms of version 2 of the GNU General Public License as
22 * published by the Free Software Foundation.
24 * In either case, Lustre is distributed in the hope that it will be
25 * useful, but WITHOUT ANY WARRANTY; without even the implied warranty
26 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
27 * license text for more details.
30 # define EXPORT_SYMTAB
32 #define DEBUG_SUBSYSTEM S_MDS
34 #include <linux/module.h>
36 #include <obd_class.h>
37 #include <lustre_ver.h>
38 #include <obd_support.h>
40 #include <lprocfs_status.h>
41 #include <lustre_mds.h>
42 #include <lustre_fid.h>
43 #include <lustre/lustre_idl.h>
45 #include "mdd_internal.h"
47 static int mdd_lov_update(struct obd_device *host,
48 struct obd_device *watched,
49 enum obd_notify_event ev, void *owner)
51 struct mdd_device *mdd = owner;
55 LASSERT(owner != NULL);
57 rc = md_do_upcall(NULL, &mdd->mdd_md_dev, MD_LOV_SYNC);
62 /* The obd is created for handling data stack for mdd */
63 int mdd_init_obd(const struct lu_env *env, struct mdd_device *mdd,
64 struct lustre_cfg *cfg)
66 char *dev = lustre_cfg_string(cfg, 0);
67 int rc, name_size, uuid_size;
70 struct lustre_cfg_bufs *bufs;
71 struct lustre_cfg *lcfg;
72 struct obd_device *obd;
75 mds_id = mdd2lu_dev(mdd)->ld_site->ls_node_id;
76 name_size = strlen(MDD_OBD_NAME) + 35;
77 uuid_size = strlen(MDD_OBD_UUID) + 35;
79 OBD_ALLOC(name, name_size);
80 OBD_ALLOC(uuid, uuid_size);
81 if (name == NULL || uuid == NULL)
82 GOTO(cleanup_mem, rc = -ENOMEM);
86 GOTO(cleanup_mem, rc = -ENOMEM);
88 snprintf(name, strlen(MDD_OBD_NAME) + 35, "%s-%s-%d",
89 MDD_OBD_NAME, dev, mds_id);
91 snprintf(uuid, strlen(MDD_OBD_UUID) + 35, "%s-%s-%d",
92 MDD_OBD_UUID, dev, mds_id);
94 lustre_cfg_bufs_reset(bufs, name);
95 lustre_cfg_bufs_set_string(bufs, 1, MDD_OBD_TYPE);
96 lustre_cfg_bufs_set_string(bufs, 2, uuid);
97 lustre_cfg_bufs_set_string(bufs, 3, (char*)dev/* MDD_OBD_PROFILE */);
98 lustre_cfg_bufs_set_string(bufs, 4, (char*)dev);
100 lcfg = lustre_cfg_new(LCFG_ATTACH, bufs);
103 GOTO(cleanup_mem, rc = -ENOMEM);
105 rc = class_attach(lcfg);
107 GOTO(lcfg_cleanup, rc);
109 obd = class_name2obd(name);
111 CERROR("Can not find obd %s\n", MDD_OBD_NAME);
115 obd->obd_recovering = 1;
116 obd->u.mds.mds_id = mds_id;
117 rc = class_setup(obd, lcfg);
119 GOTO(class_detach, rc);
122 * Add here for obd notify mechanism, when adding a new ost, the mds
123 * will notify this mdd.
125 obd->obd_upcall.onu_upcall = mdd_lov_update;
126 obd->obd_upcall.onu_owner = mdd;
127 mdd->mdd_obd_dev = obd;
132 class_detach(obd, lcfg);
134 lustre_cfg_free(lcfg);
137 OBD_FREE(name, name_size);
139 OBD_FREE(uuid, uuid_size);
143 int mdd_fini_obd(const struct lu_env *env, struct mdd_device *mdd,
144 struct lustre_cfg *lcfg)
146 struct obd_device *obd;
150 obd = mdd2obd_dev(mdd);
153 rc = class_cleanup(obd, lcfg);
155 GOTO(lcfg_cleanup, rc);
157 obd->obd_upcall.onu_upcall = NULL;
158 obd->obd_upcall.onu_owner = NULL;
159 rc = class_detach(obd, lcfg);
161 GOTO(lcfg_cleanup, rc);
162 mdd->mdd_obd_dev = NULL;
169 int mdd_get_md(const struct lu_env *env, struct mdd_object *obj,
170 void *md, int *md_size, const char *name)
175 rc = mdo_xattr_get(env, obj, mdd_buf_get(env, md, *md_size), name,
176 mdd_object_capa(env, obj));
178 * XXX: Handling of -ENODATA, the right way is to have ->do_md_get()
179 * exported by dt layer.
181 if (rc == 0 || rc == -ENODATA) {
185 CERROR("Error %d reading eadata \n", rc);
187 /* XXX: Convert lov EA but fixed after verification test. */
194 int mdd_get_md_locked(const struct lu_env *env, struct mdd_object *obj,
195 void *md, int *md_size, const char *name)
198 mdd_read_lock(env, obj);
199 rc = mdd_get_md(env, obj, md, md_size, name);
200 mdd_read_unlock(env, obj);
204 static int mdd_lov_set_stripe_md(const struct lu_env *env,
205 struct mdd_object *obj, struct lu_buf *buf,
206 struct thandle *handle)
208 struct mdd_device *mdd = mdo2mdd(&obj->mod_obj);
209 struct obd_device *obd = mdd2obd_dev(mdd);
210 struct obd_export *lov_exp = obd->u.mds.mds_osc_exp;
211 struct lov_stripe_md *lsm = NULL;
215 LASSERT(S_ISDIR(mdd_object_type(obj)) || S_ISREG(mdd_object_type(obj)));
216 rc = obd_iocontrol(OBD_IOC_LOV_SETSTRIPE, lov_exp, 0,
220 obd_free_memmd(lov_exp, &lsm);
222 rc = mdd_xattr_set_txn(env, obj, buf, MDS_LOV_MD_NAME, 0, handle);
224 CDEBUG(D_INFO, "set lov ea of "DFID" rc %d \n", PFID(mdo2fid(obj)), rc);
229 * Permission check is done before call it,
230 * no need check again.
232 static int mdd_lov_set_dir_md(const struct lu_env *env,
233 struct mdd_object *obj, struct lu_buf *buf,
234 struct thandle *handle)
236 struct lov_user_md *lum = NULL;
240 LASSERT(S_ISDIR(mdd_object_type(obj)));
241 lum = (struct lov_user_md*)buf->lb_buf;
243 /* if { size, offset, count } = { 0, -1, 0 } (i.e. all default
244 * values specified) then delete default striping from dir. */
245 if ((lum->lmm_stripe_size == 0 && lum->lmm_stripe_count == 0 &&
246 lum->lmm_stripe_offset == (typeof(lum->lmm_stripe_offset))(-1)) ||
247 /* lmm_stripe_size == -1 is deprecated in 1.4.6 */
248 lum->lmm_stripe_size == (typeof(lum->lmm_stripe_size))(-1)){
249 rc = mdd_xattr_set_txn(env, obj, &LU_BUF_NULL,
250 MDS_LOV_MD_NAME, 0, handle);
253 CDEBUG(D_INFO, "delete lov ea of "DFID" rc %d \n",
254 PFID(mdo2fid(obj)), rc);
256 rc = mdd_lov_set_stripe_md(env, obj, buf, handle);
261 int mdd_lsm_sanity_check(const struct lu_env *env, struct mdd_object *obj)
263 struct lu_attr *tmp_la = &mdd_env_info(env)->mti_la;
264 struct md_ucred *uc = md_ucred(env);
268 rc = mdd_la_get(env, obj, tmp_la, BYPASS_CAPA);
272 if ((uc->mu_fsuid != tmp_la->la_uid) && !mdd_capable(uc, CAP_FOWNER))
273 rc = mdd_permission_internal_locked(env, obj, tmp_la,
279 int mdd_lov_set_md(const struct lu_env *env, struct mdd_object *pobj,
280 struct mdd_object *child, struct lov_mds_md *lmmp,
281 int lmm_size, struct thandle *handle, int set_stripe)
288 buf = mdd_buf_get(env, lmmp, lmm_size);
289 mode = mdd_object_type(child);
290 if (S_ISREG(mode) && lmm_size > 0) {
292 rc = mdd_lov_set_stripe_md(env, child, buf, handle);
294 rc = mdd_xattr_set_txn(env, child, buf,
295 MDS_LOV_MD_NAME, 0, handle);
297 } else if (S_ISDIR(mode)) {
298 if (lmmp == NULL && lmm_size == 0) {
299 struct mdd_device *mdd = mdd_obj2mdd_dev(child);
300 struct lov_mds_md *lmm = mdd_max_lmm_get(env, mdd);
301 int size = sizeof(*lmm);
303 /* Get parent dir stripe and set */
305 rc = mdd_get_md_locked(env, pobj, lmm, &size,
308 buf = mdd_buf_get(env, lmm, size);
309 rc = mdd_xattr_set_txn(env, child, buf,
310 MDS_LOV_MD_NAME, 0, handle);
312 CERROR("error on copy stripe info: rc "
316 LASSERT(lmmp != NULL && lmm_size > 0);
317 rc = mdd_lov_set_dir_md(env, child, buf, handle);
320 CDEBUG(D_INFO, "Set lov md %p size %d for fid "DFID" rc %d\n",
321 lmmp, lmm_size, PFID(mdo2fid(child)), rc);
326 * XXX: this is for create lsm object id, which should identify the lsm object
327 * unique in the whole mds, as I see. But it seems, we still not need it
328 * now. Right? So just borrow the ll_fid_build_ino().
330 static obd_id mdd_lov_create_id(const struct lu_fid *fid)
332 return fid_flatten(fid);
335 static int mdd_lov_objid_alloc(const struct lu_env *env,
336 struct mdd_device *mdd)
338 struct mdd_thread_info *info = mdd_env_info(env);
339 struct mds_obd *mds = &mdd->mdd_obd_dev->u.mds;
341 OBD_ALLOC(info->mti_oti.oti_objid,
342 mds->mds_lov_desc.ld_tgt_count * sizeof(obd_id));
343 return (info->mti_oti.oti_objid == NULL ? -ENOMEM : 0);
346 void mdd_lov_objid_update(const struct lu_env *env, struct mdd_device *mdd)
348 struct mdd_thread_info *info = mdd_env_info(env);
349 if (info->mti_oti.oti_objid != NULL)
350 mds_lov_update_objids(mdd->mdd_obd_dev,
351 info->mti_oti.oti_objid);
354 static void mdd_lov_objid_from_lmm(const struct lu_env *env,
355 struct mdd_device *mdd,
356 struct lov_mds_md *lmm)
358 struct mds_obd *mds = &mdd->mdd_obd_dev->u.mds;
359 struct mdd_thread_info *info = mdd_env_info(env);
360 mds_objids_from_lmm(info->mti_oti.oti_objid, lmm, &mds->mds_lov_desc);
363 static void mdd_lov_objid_free(const struct lu_env *env,
364 struct mdd_device *mdd)
366 struct mdd_thread_info *info = mdd_env_info(env);
367 struct mds_obd *mds = &mdd->mdd_obd_dev->u.mds;
369 OBD_FREE(info->mti_oti.oti_objid,
370 mds->mds_lov_desc.ld_tgt_count * sizeof(obd_id));
371 info->mti_oti.oti_objid = NULL;
374 void mdd_lov_create_finish(const struct lu_env *env, struct mdd_device *mdd,
375 struct lov_mds_md *lmm, int lmm_size,
376 const struct md_op_spec *spec)
378 struct mdd_thread_info *info = mdd_env_info(env);
380 if (lmm && !spec->u.sp_ea.no_lov_create)
381 OBD_FREE(lmm, lmm_size);
383 if (info->mti_oti.oti_objid != NULL)
384 mdd_lov_objid_free(env, mdd);
387 int mdd_lov_create(const struct lu_env *env, struct mdd_device *mdd,
388 struct mdd_object *parent, struct mdd_object *child,
389 struct lov_mds_md **lmm, int *lmm_size,
390 const struct md_op_spec *spec, struct lu_attr *la)
392 struct obd_device *obd = mdd2obd_dev(mdd);
393 struct obd_export *lov_exp = obd->u.mds.mds_osc_exp;
395 struct lov_stripe_md *lsm = NULL;
396 const void *eadata = spec->u.sp_ea.eadata;
397 __u32 create_flags = spec->sp_cr_flags;
398 struct obd_trans_info *oti = &mdd_env_info(env)->mti_oti;
402 if (create_flags & MDS_OPEN_DELAY_CREATE ||
403 !(create_flags & FMODE_WRITE))
407 rc = mdd_lov_objid_alloc(env, mdd);
411 /* replay case, has objects already, only get lov from eadata */
412 if (spec->u.sp_ea.no_lov_create != 0) {
413 *lmm = (struct lov_mds_md *)spec->u.sp_ea.eadata;
414 *lmm_size = spec->u.sp_ea.eadatalen;
415 mdd_lov_objid_from_lmm(env, mdd, *lmm);
419 if (OBD_FAIL_CHECK_ONCE(OBD_FAIL_MDS_ALLOC_OBDO))
420 GOTO(out_ids, rc = -ENOMEM);
422 LASSERT(lov_exp != NULL);
423 oa = &mdd_env_info(env)->mti_oa;
425 oa->o_uid = 0; /* must have 0 uid / gid on OST */
427 oa->o_gr = FILTER_GROUP_MDS0 + mdd2lu_dev(mdd)->ld_site->ls_node_id;
428 oa->o_mode = S_IFREG | 0600;
429 oa->o_id = mdd_lov_create_id(mdd_object_fid(child));
430 oa->o_valid = OBD_MD_FLID | OBD_MD_FLTYPE | OBD_MD_FLFLAGS |
431 OBD_MD_FLMODE | OBD_MD_FLUID | OBD_MD_FLGID | OBD_MD_FLGROUP;
434 if (!(create_flags & MDS_OPEN_HAS_OBJS)) {
435 if (create_flags & MDS_OPEN_HAS_EA) {
436 LASSERT(eadata != NULL);
437 rc = obd_iocontrol(OBD_IOC_LOV_SETSTRIPE, lov_exp,
438 0, &lsm, (void*)eadata);
441 lsm->lsm_object_id = oa->o_id;
442 lsm->lsm_object_gr = oa->o_gr;
443 } else if (parent != NULL) {
444 /* get lov ea from parent and set to lov */
445 struct lov_mds_md *_lmm;
448 _lmm_size = mdd_lov_mdsize(env, mdd);
449 _lmm = mdd_max_lmm_get(env, mdd);
452 GOTO(out_oti, rc = -ENOMEM);
454 rc = mdd_get_md_locked(env, parent, _lmm,
458 rc = obd_iocontrol(OBD_IOC_LOV_SETSTRIPE,
459 lov_exp, 0, &lsm, _lmm);
464 rc = obd_create(lov_exp, oa, &lsm, oti);
467 CERROR("Create error for "DFID": %d\n",
468 PFID(mdo2fid(child)), rc);
473 LASSERT(lsm->lsm_object_gr >= FILTER_GROUP_MDS0);
475 LASSERT(eadata != NULL);
476 rc = obd_iocontrol(OBD_IOC_LOV_SETEA, lov_exp, 0, &lsm,
480 lsm->lsm_object_id = oa->o_id;
481 lsm->lsm_object_gr = oa->o_gr;
485 * Sometimes, we may truncate some object(without lsm) then open it
486 * (with write flags), so creating lsm above. The Nonzero(truncated)
487 * size should tell ost, since size attr is in charge by OST.
489 if (la->la_size && la->la_valid & LA_SIZE) {
490 struct obd_info *oinfo = &mdd_env_info(env)->mti_oi;
492 memset(oinfo, 0, sizeof(*oinfo));
494 oa->o_size = la->la_size;
496 /* When setting attr to ost, FLBKSZ is not needed. */
497 oa->o_valid &= ~OBD_MD_FLBLKSZ;
498 obdo_from_la(oa, la, OBD_MD_FLTYPE | OBD_MD_FLATIME |
499 OBD_MD_FLMTIME | OBD_MD_FLCTIME | OBD_MD_FLSIZE);
502 * XXX: Pack lustre id to OST, in OST, it will be packed by
503 * filter_fid, but can not see what is the usages. So just pack
504 * o_seq o_ver here, maybe fix it after this cycle.
506 oa->o_fid = fid_seq(mdd_object_fid(child));
507 oa->o_generation = fid_oid(mdd_object_fid(child));
508 oa->o_valid |= OBD_MD_FLFID | OBD_MD_FLGENER;
511 oinfo->oi_capa = mdo_capa_get(env, child, NULL, 0,
512 CAPA_OPC_MDS_DEFAULT);
513 if (IS_ERR(oinfo->oi_capa))
514 oinfo->oi_capa = NULL;
516 rc = obd_setattr(lov_exp, oinfo, oti);
517 capa_put(oinfo->oi_capa);
519 CERROR("Error setting attrs for "DFID": rc %d\n",
520 PFID(mdo2fid(child)), rc);
522 CERROR("obd_setattr for "DFID" rc %d\n",
523 PFID(mdo2fid(child)), rc);
530 /* blksize should be changed after create data object */
531 la->la_valid |= LA_BLKSIZE;
532 la->la_blksize = oa->o_blksize;
534 rc = obd_packmd(lov_exp, lmm, lsm);
536 CERROR("Cannot pack lsm, err = %d\n", rc);
543 oti_free_cookies(oti);
546 obd_free_memmd(lov_exp, &lsm);
548 mdd_lov_objid_free(env, mdd);
555 * used when destroying orphans and from mds_reint_unlink() when MDS wants to
556 * destroy objects on OSS.
559 int mdd_lovobj_unlink(const struct lu_env *env, struct mdd_device *mdd,
560 struct mdd_object *obj, struct lu_attr *la,
561 struct lov_mds_md *lmm, int lmm_size,
562 struct llog_cookie *logcookies,
565 struct obd_device *obd = mdd2obd_dev(mdd);
566 struct obd_export *lov_exp = obd->u.mds.mds_osc_exp;
567 struct lov_stripe_md *lsm = NULL;
568 struct obd_trans_info *oti = &mdd_env_info(env)->mti_oti;
569 struct obdo *oa = &mdd_env_info(env)->mti_oa;
576 rc = obd_unpackmd(lov_exp, &lsm, lmm, lmm_size);
578 CERROR("Error unpack md %p\n", lmm);
581 LASSERT(rc >= sizeof(*lsm));
585 oa->o_id = lsm->lsm_object_id;
586 oa->o_gr = FILTER_GROUP_MDS0 + mdd2lu_dev(mdd)->ld_site->ls_node_id;
587 oa->o_mode = la->la_mode & S_IFMT;
588 oa->o_valid = OBD_MD_FLID | OBD_MD_FLTYPE | OBD_MD_FLGROUP;
591 if (log_unlink && logcookies) {
592 oa->o_valid |= OBD_MD_FLCOOKIE;
593 oti->oti_logcookies = logcookies;
596 CDEBUG(D_INFO, "destroying OSS object %d/%d\n",
597 (int)oa->o_id, (int)oa->o_gr);
599 rc = obd_destroy(lov_exp, oa, lsm, oti, NULL);
601 obd_free_memmd(lov_exp, &lsm);
607 * called with obj not locked.
609 int mdd_lov_destroy(const struct lu_env *env, struct mdd_device *mdd,
610 struct mdd_object *obj, struct lu_attr *la)
612 struct md_attr *ma = &mdd_env_info(env)->mti_ma;
616 if (unlikely(la->la_nlink != 0)) {
617 CWARN("Attempt to destroy OSS object when nlink == %d\n",
622 ma->ma_lmm_size = mdd_lov_mdsize(env, mdd);
623 ma->ma_lmm = mdd_max_lmm_get(env, mdd);
624 ma->ma_cookie_size = mdd_lov_cookiesize(env, mdd);
625 ma->ma_cookie = mdd_max_cookie_get(env, mdd);
626 if (ma->ma_lmm == NULL || ma->ma_cookie == NULL)
627 RETURN(rc = -ENOMEM);
630 rc = mdd_get_md_locked(env, obj, ma->ma_lmm, &ma->ma_lmm_size,
633 CWARN("Get lov ea failed for "DFID"\n", PFID(mdo2fid(obj)));
636 ma->ma_valid = MA_LOV;
638 rc = mdd_unlink_log(env, mdd, obj, ma);
640 CWARN("mds unlink log for "DFID" failed: %d\n",
641 PFID(mdo2fid(obj)), rc);
644 if (ma->ma_valid | MA_COOKIE)
645 rc = mdd_lovobj_unlink(env, mdd, obj, la,
646 ma->ma_lmm, ma->ma_lmm_size,
652 int mdd_unlink_log(const struct lu_env *env, struct mdd_device *mdd,
653 struct mdd_object *mdd_cobj, struct md_attr *ma)
655 struct obd_device *obd = mdd2obd_dev(mdd);
657 LASSERT(ma->ma_valid & MA_LOV);
659 if ((ma->ma_cookie_size > 0) &&
660 (mds_log_op_unlink(obd, ma->ma_lmm, ma->ma_lmm_size,
661 ma->ma_cookie, ma->ma_cookie_size) > 0)) {
662 ma->ma_valid |= MA_COOKIE;
667 int mdd_setattr_log(const struct lu_env *env, struct mdd_device *mdd,
668 const struct md_attr *ma,
669 struct lov_mds_md *lmm, int lmm_size,
670 struct llog_cookie *logcookies, int cookies_size)
672 struct obd_device *obd = mdd2obd_dev(mdd);
674 /* journal chown/chgrp in llog, just like unlink */
676 CDEBUG(D_INFO, "setattr llog for uid/gid=%lu/%lu\n",
677 (unsigned long)ma->ma_attr.la_uid,
678 (unsigned long)ma->ma_attr.la_gid);
679 return mds_log_op_setattr(obd, ma->ma_attr.la_uid,
680 ma->ma_attr.la_gid, lmm,
681 lmm_size, logcookies,
687 int mdd_lov_setattr_async(const struct lu_env *env, struct mdd_object *obj,
688 struct lov_mds_md *lmm, int lmm_size,
689 struct llog_cookie *logcookies)
691 struct mdd_device *mdd = mdo2mdd(&obj->mod_obj);
692 struct obd_device *obd = mdd2obd_dev(mdd);
693 struct lu_attr *tmp_la = &mdd_env_info(env)->mti_la;
694 const struct lu_fid *fid = mdd_object_fid(obj);
699 mdd_read_lock(env, obj);
700 rc = mdo_attr_get(env, obj, tmp_la, mdd_object_capa(env, obj));
701 mdd_read_unlock(env, obj);
705 oc = mdo_capa_get(env, obj, NULL, 0, CAPA_OPC_MDS_DEFAULT);
709 rc = mds_osc_setattr_async(obd, tmp_la->la_uid, tmp_la->la_gid, lmm,
710 lmm_size, logcookies, fid_seq(fid),