1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
5 * Lustre Metadata Server (mds) handling of striped file data
7 * Copyright (C) 2001-2003 Cluster File Systems, Inc.
8 * Author: Peter Braam <braam@clusterfs.com>
10 * This file is part of Lustre, http://www.lustre.org.
12 * Lustre is free software; you can redistribute it and/or
13 * modify it under the terms of version 2 of the GNU General Public
14 * License as published by the Free Software Foundation.
16 * Lustre is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License
22 * along with Lustre; if not, write to the Free Software
23 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
27 # define EXPORT_SYMTAB
29 #define DEBUG_SUBSYSTEM S_MDS
31 #include <linux/module.h>
32 #include <linux/lustre_mds.h>
33 #include <linux/lustre_idl.h>
34 #include <linux/obd_class.h>
35 #include <linux/obd_lov.h>
36 #include <linux/lustre_lib.h>
37 #include <linux/lustre_fsfilt.h>
39 #include "mds_internal.h"
41 void mds_lov_update_objids(struct obd_device *obd, obd_id *ids)
43 struct mds_obd *mds = &obd->u.mds;
48 for (i = 0; i < mds->mds_lov_desc.ld_tgt_count; i++)
49 if (ids[i] > (mds->mds_lov_objids)[i])
50 (mds->mds_lov_objids)[i] = ids[i];
55 static int mds_lov_read_objids(struct obd_device *obd)
57 struct mds_obd *mds = &obd->u.mds;
60 int i, rc, size = mds->mds_lov_desc.ld_tgt_count * sizeof(*ids);
63 if (mds->mds_lov_objids != NULL)
69 mds->mds_lov_objids = ids;
71 if (mds->mds_lov_objid_filp->f_dentry->d_inode->i_size == 0)
73 rc = fsfilt_read_record(obd, mds->mds_lov_objid_filp, ids, size, &off);
75 CERROR("Error reading objids %d\n", rc);
77 mds->mds_lov_objids_valid = 1;
81 for (i = 0; i < mds->mds_lov_desc.ld_tgt_count; i++)
82 CDEBUG(D_INFO, "read last object "LPU64" for idx %d\n",
83 mds->mds_lov_objids[i], i);
88 int mds_lov_write_objids(struct obd_device *obd)
90 struct mds_obd *mds = &obd->u.mds;
92 int i, rc, size = mds->mds_lov_desc.ld_tgt_count * sizeof(obd_id);
95 for (i = 0; i < mds->mds_lov_desc.ld_tgt_count; i++)
96 CDEBUG(D_INFO, "writing last object "LPU64" for idx %d\n",
97 mds->mds_lov_objids[i], i);
99 rc = fsfilt_write_record(obd, mds->mds_lov_objid_filp,
100 mds->mds_lov_objids, size, &off, 0);
104 int mds_lov_clearorphans(struct mds_obd *mds, struct obd_uuid *ost_uuid)
108 struct obd_trans_info oti = {0};
109 struct lov_stripe_md *empty_ea = NULL;
112 LASSERT(mds->mds_lov_objids != NULL);
114 /* This create will in fact either create or destroy: If the OST is
115 * missing objects below this ID, they will be created. If it finds
116 * objects above this ID, they will be removed. */
117 memset(&oa, 0, sizeof(oa));
118 oa.o_valid = OBD_MD_FLFLAGS;
119 oa.o_flags = OBD_FL_DELORPHAN;
120 if (ost_uuid != NULL) {
121 memcpy(&oa.o_inline, ost_uuid, sizeof(*ost_uuid));
122 oa.o_valid |= OBD_MD_FLINLINE;
124 rc = obd_create(mds->mds_osc_exp, &oa, &empty_ea, &oti);
129 /* update the LOV-OSC knowledge of the last used object id's */
130 int mds_lov_set_nextid(struct obd_device *obd)
132 struct mds_obd *mds = &obd->u.mds;
136 LASSERT(!obd->obd_recovering);
138 LASSERT(mds->mds_lov_objids != NULL);
140 rc = obd_set_info(mds->mds_osc_exp, strlen("next_id"), "next_id",
141 mds->mds_lov_desc.ld_tgt_count, mds->mds_lov_objids);
145 int mds_lov_connect(struct obd_device *obd, char * lov_name)
147 struct mds_obd *mds = &obd->u.mds;
148 struct lustre_handle conn = {0,};
153 if (IS_ERR(mds->mds_osc_obd))
154 RETURN(PTR_ERR(mds->mds_osc_obd));
156 if (mds->mds_osc_obd)
159 mds->mds_osc_obd = class_name2obd(lov_name);
160 if (!mds->mds_osc_obd) {
161 CERROR("MDS cannot locate LOV %s\n", lov_name);
162 mds->mds_osc_obd = ERR_PTR(-ENOTCONN);
166 rc = obd_connect(&conn, mds->mds_osc_obd, &obd->obd_uuid,
167 NULL /* obd_connect_data */);
169 CERROR("MDS cannot connect to LOV %s (%d)\n", lov_name, rc);
170 mds->mds_osc_obd = ERR_PTR(rc);
173 mds->mds_osc_exp = class_conn2export(&conn);
175 rc = obd_register_observer(mds->mds_osc_obd, obd);
177 CERROR("MDS cannot register as observer of LOV %s (%d)\n",
179 GOTO(err_discon, rc);
182 valsize = sizeof(mds->mds_lov_desc);
183 rc = obd_get_info(mds->mds_osc_exp, strlen("lovdesc") + 1, "lovdesc",
184 &valsize, &mds->mds_lov_desc);
188 mds->mds_max_mdsize = lov_mds_md_size(mds->mds_lov_desc.ld_tgt_count);
189 mds->mds_max_cookiesize = mds->mds_lov_desc.ld_tgt_count*
190 sizeof(struct llog_cookie);
191 mds->mds_has_lov_desc = 1;
192 rc = mds_lov_read_objids(obd);
194 CERROR("cannot read %s: rc = %d\n", "lov_objids", rc);
198 rc = llog_cat_initialize(obd, mds->mds_lov_desc.ld_tgt_count);
200 CERROR("failed to initialize catalog %d\n", rc);
204 /* If we're mounting this code for the first time on an existing FS,
205 * we need to populate the objids array from the real OST values */
206 if (!mds->mds_lov_objids_valid) {
207 int size = sizeof(obd_id) * mds->mds_lov_desc.ld_tgt_count;
208 rc = obd_get_info(mds->mds_osc_exp, strlen("last_id"),
209 "last_id", &size, mds->mds_lov_objids);
211 for (i = 0; i < mds->mds_lov_desc.ld_tgt_count; i++)
212 CWARN("got last object "LPU64" from OST %d\n",
213 mds->mds_lov_objids[i], i);
214 mds->mds_lov_objids_valid = 1;
215 rc = mds_lov_write_objids(obd);
217 CERROR("got last objids from OSTs, but error "
218 "writing objids file: %d\n", rc);
222 /* I want to see a callback happen when the OBD moves to a
223 * "For General Use" state, and that's when we'll call
224 * set_nextid(). The class driver can help us here, because
225 * it can use the obd_recovering flag to determine when the
226 * the OBD is full available. */
227 if (!obd->obd_recovering)
228 rc = mds_postrecov(obd);
232 obd_register_observer(mds->mds_osc_obd, NULL);
234 obd_disconnect(mds->mds_osc_exp);
235 mds->mds_osc_exp = NULL;
236 mds->mds_osc_obd = ERR_PTR(rc);
240 int mds_lov_disconnect(struct obd_device *obd)
242 struct mds_obd *mds = &obd->u.mds;
246 if (!IS_ERR(mds->mds_osc_obd) && mds->mds_osc_exp != NULL) {
247 obd_register_observer(mds->mds_osc_obd, NULL);
249 /* The actual disconnect of the mds_lov will be called from
250 * class_disconnect_exports from mds_lov_clean. So we have to
251 * ensure that class_cleanup doesn't fail due to the extra ref
252 * we're holding now. The mechanism to do that already exists -
253 * the obd_force flag. We'll drop the final ref to the
254 * mds_osc_exp in mds_cleanup. */
255 mds->mds_osc_obd->obd_force = 1;
261 /* for consistency, let's make the lov and the lov's
262 * osc's see the same cleanup flags as our mds */
263 void mds_lov_set_cleanup_flags(struct obd_device *obd)
265 struct mds_obd *mds = &obd->u.mds;
268 if (IS_ERR(mds->mds_osc_obd) || (mds->mds_osc_exp == NULL))
271 lov = &mds->mds_osc_obd->u.lov;
272 mds->mds_osc_obd->obd_force = obd->obd_force;
273 mds->mds_osc_obd->obd_fail = obd->obd_fail;
275 struct obd_export *osc_exp;
277 spin_lock(&lov->lov_lock);
278 for (i = 0; i < lov->desc.ld_tgt_count; i++) {
279 if (lov->tgts[i].ltd_exp != NULL) {
280 osc_exp = lov->tgts[i].ltd_exp;
281 osc_exp->exp_obd->obd_force = obd->obd_force;
282 osc_exp->exp_obd->obd_fail = obd->obd_fail;
285 spin_unlock(&lov->lov_lock);
289 int mds_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
290 void *karg, void *uarg)
292 static struct obd_uuid cfg_uuid = { .uuid = "config_uuid" };
293 struct obd_device *obd = exp->exp_obd;
294 struct mds_obd *mds = &obd->u.mds;
295 struct obd_ioctl_data *data = karg;
296 struct lvfs_run_ctxt saved;
300 CDEBUG(D_IOCTL, "handling ioctl cmd %#x\n", cmd);
303 case OBD_IOC_RECORD: {
304 char *name = data->ioc_inlbuf1;
305 if (mds->mds_cfg_llh)
308 push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
309 rc = llog_create(llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT),
310 &mds->mds_cfg_llh, NULL, name);
312 llog_init_handle(mds->mds_cfg_llh, LLOG_F_IS_PLAIN,
315 mds->mds_cfg_llh = NULL;
316 pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
321 case OBD_IOC_ENDRECORD: {
322 if (!mds->mds_cfg_llh)
325 push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
326 rc = llog_close(mds->mds_cfg_llh);
327 pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
329 mds->mds_cfg_llh = NULL;
333 case OBD_IOC_CLEAR_LOG: {
334 char *name = data->ioc_inlbuf1;
335 if (mds->mds_cfg_llh)
338 push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
339 rc = llog_create(llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT),
340 &mds->mds_cfg_llh, NULL, name);
342 llog_init_handle(mds->mds_cfg_llh, LLOG_F_IS_PLAIN,
345 rc = llog_destroy(mds->mds_cfg_llh);
346 llog_free_handle(mds->mds_cfg_llh);
348 pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
350 mds->mds_cfg_llh = NULL;
354 case OBD_IOC_DORECORD: {
356 struct llog_rec_hdr rec;
357 if (!mds->mds_cfg_llh)
360 rec.lrh_len = llog_data_len(data->ioc_plen1);
362 if (data->ioc_type == LUSTRE_CFG_TYPE) {
363 rec.lrh_type = OBD_CFG_REC;
365 CERROR("unknown cfg record type:%d \n", data->ioc_type);
369 OBD_ALLOC(cfg_buf, data->ioc_plen1);
372 rc = copy_from_user(cfg_buf, data->ioc_pbuf1, data->ioc_plen1);
374 OBD_FREE(cfg_buf, data->ioc_plen1);
378 push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
379 rc = llog_write_rec(mds->mds_cfg_llh, &rec, NULL, 0,
381 pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
383 OBD_FREE(cfg_buf, data->ioc_plen1);
387 case OBD_IOC_PARSE: {
388 struct llog_ctxt *ctxt =
389 llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT);
390 push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
391 rc = class_config_parse_llog(ctxt, data->ioc_inlbuf1, NULL);
392 pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
399 case OBD_IOC_DUMP_LOG: {
400 struct llog_ctxt *ctxt =
401 llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT);
402 push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
403 rc = class_config_dump_llog(ctxt, data->ioc_inlbuf1, NULL);
404 pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
412 CDEBUG(D_HA, "syncing mds %s\n", obd->obd_name);
413 rc = fsfilt_sync(obd, obd->u.mds.mds_sb);
417 case OBD_IOC_SET_READONLY: {
419 struct inode *inode = obd->u.mds.mds_sb->s_root->d_inode;
420 BDEVNAME_DECLARE_STORAGE(tmp);
421 CERROR("*** setting device %s read-only ***\n",
422 ll_bdevname(obd->u.mds.mds_sb, tmp));
424 handle = fsfilt_start(obd, inode, FSFILT_OP_MKNOD, NULL);
426 rc = fsfilt_commit(obd, inode, handle, 1);
428 CDEBUG(D_HA, "syncing mds %s\n", obd->obd_name);
429 rc = fsfilt_sync(obd, obd->u.mds.mds_sb);
431 lvfs_set_rdonly(lvfs_sbdev(obd->u.mds.mds_sb));
435 case OBD_IOC_CATLOGLIST: {
436 int count = mds->mds_lov_desc.ld_tgt_count;
437 rc = llog_catalog_list(obd, count, data);
441 case OBD_IOC_LLOG_CHECK:
442 case OBD_IOC_LLOG_CANCEL:
443 case OBD_IOC_LLOG_REMOVE: {
444 struct llog_ctxt *ctxt =
445 llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT);
448 obd_llog_finish(obd, mds->mds_lov_desc.ld_tgt_count);
449 push_ctxt(&saved, &ctxt->loc_exp->exp_obd->obd_lvfs_ctxt, NULL);
450 rc = llog_ioctl(ctxt, cmd, data);
451 pop_ctxt(&saved, &ctxt->loc_exp->exp_obd->obd_lvfs_ctxt, NULL);
452 llog_cat_initialize(obd, mds->mds_lov_desc.ld_tgt_count);
453 rc2 = obd_set_info(mds->mds_osc_exp, strlen("mds_conn"),
454 "mds_conn", 0, NULL);
459 case OBD_IOC_LLOG_INFO:
460 case OBD_IOC_LLOG_PRINT: {
461 struct llog_ctxt *ctxt =
462 llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT);
464 push_ctxt(&saved, &ctxt->loc_exp->exp_obd->obd_lvfs_ctxt, NULL);
465 rc = llog_ioctl(ctxt, cmd, data);
466 pop_ctxt(&saved, &ctxt->loc_exp->exp_obd->obd_lvfs_ctxt, NULL);
471 case OBD_IOC_ABORT_RECOVERY:
472 CERROR("aborting recovery for device %s\n", obd->obd_name);
473 target_abort_recovery(obd);
477 CDEBUG(D_INFO, "unknown command %x\n", cmd);
484 struct mds_lov_sync_info {
485 struct obd_device *mlsi_obd; /* the lov device to sync */
486 struct obd_uuid *mlsi_uuid; /* target to sync */
489 int mds_lov_synchronize(void *data)
491 struct mds_lov_sync_info *mlsi = data;
492 struct obd_device *obd;
493 struct obd_uuid *uuid;
500 SIGNAL_MASK_LOCK(current, flags);
501 sigfillset(¤t->blocked);
503 SIGNAL_MASK_UNLOCK(current, flags);
506 obd = mlsi->mlsi_obd;
507 uuid = mlsi->mlsi_uuid;
509 OBD_FREE(mlsi, sizeof(*mlsi));
511 LASSERT(obd != NULL);
512 LASSERT(uuid != NULL);
514 rc = obd_set_info(obd->u.mds.mds_osc_exp, strlen("mds_conn"),
515 "mds_conn", 0, uuid);
519 rc = llog_connect(llog_get_context(obd, LLOG_MDS_OST_ORIG_CTXT),
520 obd->u.mds.mds_lov_desc.ld_tgt_count,
523 CERROR("%s: failed at llog_origin_connect: %d\n",
528 CWARN("MDS %s: %s now active, resetting orphans\n",
529 obd->obd_name, uuid->uuid);
530 rc = mds_lov_clearorphans(&obd->u.mds, uuid);
532 CERROR("%s: failed at mds_lov_clearorphans: %d\n",
538 class_export_put(obd->obd_self_export);
542 int mds_lov_start_synchronize(struct obd_device *obd, struct obd_uuid *uuid)
544 struct mds_lov_sync_info *mlsi;
549 OBD_ALLOC(mlsi, sizeof(*mlsi));
553 mlsi->mlsi_obd = obd;
554 mlsi->mlsi_uuid = uuid;
556 /* We need to lock the mds in place for our new thread context. */
557 class_export_get(obd->obd_self_export);
559 rc = kernel_thread(mds_lov_synchronize, mlsi, CLONE_VM | CLONE_FILES);
561 CERROR("%s: error starting mds_lov_synchronize: %d\n",
564 CDEBUG(D_HA, "%s: mds_lov_synchronize thread: %d\n",
572 int mds_notify(struct obd_device *obd, struct obd_device *watched, int active)
574 struct obd_uuid *uuid;
581 if (strcmp(watched->obd_type->typ_name, "osc")) {
582 CERROR("unexpected notification of %s %s!\n",
583 watched->obd_type->typ_name, watched->obd_name);
587 uuid = &watched->u.cli.cl_import->imp_target_uuid;
588 if (obd->obd_recovering) {
589 CWARN("MDS %s: in recovery, not resetting orphans on %s\n",
590 obd->obd_name, uuid->uuid);
592 LASSERT(llog_get_context(obd, LLOG_MDS_OST_ORIG_CTXT) != NULL);
594 rc = mds_lov_start_synchronize(obd, uuid);
599 /* Convert the on-disk LOV EA structre.
600 * We always try to convert from an old LOV EA format to the common in-memory
601 * (lsm) format (obd_unpackmd() understands the old on-disk (lmm) format) and
602 * then convert back to the new on-disk format and save it back to disk
603 * (obd_packmd() only ever saves to the new on-disk format) so we don't have
604 * to convert it each time this inode is accessed.
606 * This function is a bit interesting in the error handling. We can safely
607 * ship the old lmm to the client in case of failure, since it uses the same
608 * obd_unpackmd() code and can do the conversion if the MDS fails for some
609 * reason. We will not delete the old lmm data until we have written the
610 * new format lmm data in fsfilt_set_md(). */
611 int mds_convert_lov_ea(struct obd_device *obd, struct inode *inode,
612 struct lov_mds_md *lmm, int lmm_size)
614 struct lov_stripe_md *lsm = NULL;
619 if (le32_to_cpu(lmm->lmm_magic) == LOV_MAGIC)
622 CDEBUG(D_INODE, "converting LOV EA on %lu/%u from %#08x to %#08x\n",
623 inode->i_ino, inode->i_generation, le32_to_cpu(lmm->lmm_magic),
625 rc = obd_unpackmd(obd->u.mds.mds_osc_exp, &lsm, lmm, lmm_size);
629 rc = obd_packmd(obd->u.mds.mds_osc_exp, &lmm, lsm);
634 handle = fsfilt_start(obd, inode, FSFILT_OP_SETATTR, NULL);
635 if (IS_ERR(handle)) {
636 rc = PTR_ERR(handle);
640 rc = fsfilt_set_md(obd, inode, handle, lmm, lmm_size);
642 err = fsfilt_commit(obd, inode, handle, 0);
644 rc = err ? err : lmm_size;
647 obd_free_memmd(obd->u.mds.mds_osc_exp, &lsm);
652 void mds_objids_from_lmm(obd_id *ids, struct lov_mds_md *lmm,
653 struct lov_desc *desc)
656 for (i = 0; i < le32_to_cpu(lmm->lmm_stripe_count); i++) {
657 ids[le32_to_cpu(lmm->lmm_objects[i].l_ost_idx)] =
658 le64_to_cpu(lmm->lmm_objects[i].l_object_id);