1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
5 * Lustre Metadata Server (mds) handling of striped file data
7 * Copyright (C) 2001-2003 Cluster File Systems, Inc.
8 * Author: Peter Braam <braam@clusterfs.com>
10 * This file is part of Lustre, http://www.lustre.org.
12 * Lustre is free software; you can redistribute it and/or
13 * modify it under the terms of version 2 of the GNU General Public
14 * License as published by the Free Software Foundation.
16 * Lustre is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License
22 * along with Lustre; if not, write to the Free Software
23 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
27 # define EXPORT_SYMTAB
29 #define DEBUG_SUBSYSTEM S_MDS
31 #include <linux/module.h>
32 #include <linux/lustre_mds.h>
33 #include <linux/obd_ost.h>
34 #include <linux/lustre_idl.h>
35 #include <linux/obd_class.h>
36 #include <linux/obd_lov.h>
37 #include <linux/lustre_lib.h>
38 #include <linux/lustre_fsfilt.h>
40 #include "mds_internal.h"
42 void le_lov_desc_to_cpu (struct lov_desc *ld)
44 ld->ld_tgt_count = le32_to_cpu (ld->ld_tgt_count);
45 ld->ld_default_stripe_count = le32_to_cpu (ld->ld_default_stripe_count);
46 ld->ld_default_stripe_size = le32_to_cpu (ld->ld_default_stripe_size);
47 ld->ld_pattern = le32_to_cpu (ld->ld_pattern);
50 void cpu_to_le_lov_desc (struct lov_desc *ld)
52 ld->ld_tgt_count = cpu_to_le32 (ld->ld_tgt_count);
53 ld->ld_default_stripe_count = cpu_to_le32 (ld->ld_default_stripe_count);
54 ld->ld_default_stripe_size = cpu_to_le32 (ld->ld_default_stripe_size);
55 ld->ld_pattern = cpu_to_le32 (ld->ld_pattern);
58 void mds_lov_update_objids(struct obd_device *obd, obd_id *ids)
60 struct mds_obd *mds = &obd->u.mds;
65 for (i = 0; i < mds->mds_lov_desc.ld_tgt_count; i++)
66 if (ids[i] > (mds->mds_lov_objids)[i])
67 (mds->mds_lov_objids)[i] = ids[i];
72 static int mds_lov_read_objids(struct obd_device *obd)
74 struct mds_obd *mds = &obd->u.mds;
77 int i, rc, size = mds->mds_lov_desc.ld_tgt_count * sizeof(*ids);
80 if (mds->mds_lov_objids != NULL)
86 mds->mds_lov_objids = ids;
88 if (mds->mds_lov_objid_filp->f_dentry->d_inode->i_size == 0)
90 rc = fsfilt_read_record(obd, mds->mds_lov_objid_filp, ids, size, &off);
92 CERROR("Error reading objids %d\n", rc);
94 mds->mds_lov_objids_valid = 1;
98 for (i = 0; i < mds->mds_lov_desc.ld_tgt_count; i++)
99 CDEBUG(D_INFO, "read last object "LPU64" for idx %d\n",
100 mds->mds_lov_objids[i], i);
105 int mds_lov_write_objids(struct obd_device *obd)
107 struct mds_obd *mds = &obd->u.mds;
109 int i, rc, size = mds->mds_lov_desc.ld_tgt_count * sizeof(obd_id);
112 for (i = 0; i < mds->mds_lov_desc.ld_tgt_count; i++)
113 CDEBUG(D_INFO, "writing last object "LPU64" for idx %d\n",
114 mds->mds_lov_objids[i], i);
116 rc = fsfilt_write_record(obd, mds->mds_lov_objid_filp,
117 mds->mds_lov_objids, size, &off, 0);
121 int mds_lov_clearorphans(struct mds_obd *mds, struct obd_uuid *ost_uuid)
125 struct obd_trans_info oti = {0};
126 struct lov_stripe_md *empty_ea = NULL;
129 LASSERT(mds->mds_lov_objids != NULL);
131 /* This create will in fact either create or destroy: If the OST is
132 * missing objects below this ID, they will be created. If it finds
133 * objects above this ID, they will be removed. */
134 memset(&oa, 0, sizeof(oa));
135 oa.o_gr = FILTER_GROUP_FIRST_MDS + mds->mds_num;
136 oa.o_valid = OBD_MD_FLFLAGS | OBD_MD_FLGROUP;
137 oa.o_flags = OBD_FL_DELORPHAN;
138 if (ost_uuid != NULL) {
139 memcpy(&oa.o_inline, ost_uuid, sizeof(*ost_uuid));
140 oa.o_valid |= OBD_MD_FLINLINE;
142 rc = obd_create(mds->mds_osc_exp, &oa, &empty_ea, &oti);
147 /* update the LOV-OSC knowledge of the last used object id's */
148 int mds_lov_set_nextid(struct obd_device *obd)
150 struct mds_obd *mds = &obd->u.mds;
154 LASSERT(!obd->obd_recovering);
156 LASSERT(mds->mds_lov_objids != NULL);
158 rc = obd_set_info(mds->mds_osc_exp, strlen("next_id"), "next_id",
159 mds->mds_lov_desc.ld_tgt_count, mds->mds_lov_objids);
163 /* tell the LOV-OSC by how much to pre-create */
164 int mds_lov_set_growth(struct mds_obd *mds, int count)
169 rc = obd_set_info(mds->mds_osc_exp, strlen("growth_count"),
170 "growth_count", sizeof(count), &count);
175 int mds_lov_connect(struct obd_device *obd, char * lov_name)
177 struct mds_obd *mds = &obd->u.mds;
178 struct lustre_handle conn = {0,};
179 char name[32] = "CATLIST";
184 if (IS_ERR(mds->mds_osc_obd))
185 RETURN(PTR_ERR(mds->mds_osc_obd));
187 if (mds->mds_osc_obd)
190 mds->mds_osc_obd = class_name2obd(lov_name);
191 if (!mds->mds_osc_obd) {
192 CERROR("MDS cannot locate LOV %s\n",
194 mds->mds_osc_obd = ERR_PTR(-ENOTCONN);
198 rc = obd_connect(&conn, mds->mds_osc_obd, &obd->obd_uuid);
200 CERROR("MDS cannot connect to LOV %s (%d)\n",
202 mds->mds_osc_obd = ERR_PTR(rc);
205 mds->mds_osc_exp = class_conn2export(&conn);
207 rc = obd_register_observer(mds->mds_osc_obd, obd);
209 CERROR("MDS cannot register as observer of LOV %s (%d)\n",
211 GOTO(err_discon, rc);
214 valsize = sizeof(mds->mds_lov_desc);
215 rc = obd_get_info(mds->mds_osc_exp, strlen("lovdesc") + 1, "lovdesc",
216 &valsize, &mds->mds_lov_desc);
220 i = lov_mds_md_size(mds->mds_lov_desc.ld_tgt_count);
221 if (i > mds->mds_max_mdsize)
222 mds->mds_max_mdsize = i;
223 mds->mds_max_cookiesize = mds->mds_lov_desc.ld_tgt_count*
224 sizeof(struct llog_cookie);
225 mds->mds_has_lov_desc = 1;
226 rc = mds_lov_read_objids(obd);
228 CERROR("cannot read %s: rc = %d\n", "lov_objids", rc);
232 rc = obd_llog_cat_initialize(obd, &obd->obd_llogs,
233 mds->mds_lov_desc.ld_tgt_count, name);
235 CERROR("failed to initialize catalog %d\n", rc);
239 /* FIXME before set info call is made, we must initialize logging */
240 group = FILTER_GROUP_FIRST_MDS + mds->mds_num;
241 valsize = sizeof(group);
242 rc = obd_set_info(mds->mds_osc_exp, strlen("mds_conn"), "mds_conn",
247 /* If we're mounting this code for the first time on an existing FS,
248 * we need to populate the objids array from the real OST values */
249 if (!mds->mds_lov_objids_valid) {
250 int size = sizeof(obd_id) * mds->mds_lov_desc.ld_tgt_count;
251 rc = obd_get_info(mds->mds_osc_exp, strlen("last_id"),
252 "last_id", &size, mds->mds_lov_objids);
254 for (i = 0; i < mds->mds_lov_desc.ld_tgt_count; i++)
255 CWARN("got last object "LPU64" from OST %d\n",
256 mds->mds_lov_objids[i], i);
257 mds->mds_lov_objids_valid = 1;
258 rc = mds_lov_write_objids(obd);
260 CERROR("got last objids from OSTs, but error "
261 "writing objids file: %d\n", rc);
265 /* I want to see a callback happen when the OBD moves to a
266 * "For General Use" state, and that's when we'll call
267 * set_nextid(). The class driver can help us here, because
268 * it can use the obd_recovering flag to determine when the
269 * the OBD is full available. */
270 if (!obd->obd_recovering)
271 rc = mds_postrecov(obd);
275 obd_register_observer(mds->mds_osc_obd, NULL);
277 obd_disconnect(mds->mds_osc_exp, 0);
278 mds->mds_osc_exp = NULL;
279 mds->mds_osc_obd = ERR_PTR(rc);
283 int mds_lov_disconnect(struct obd_device *obd, int flags)
285 struct mds_obd *mds = &obd->u.mds;
289 if (!IS_ERR(mds->mds_osc_obd) && mds->mds_osc_exp != NULL) {
290 /* cleanup all llogging subsystems */
291 rc = obd_llog_finish(obd, &obd->obd_llogs,
292 mds->mds_lov_desc.ld_tgt_count);
294 CERROR("failed to cleanup llogging subsystems\n");
296 obd_register_observer(mds->mds_osc_obd, NULL);
298 rc = obd_disconnect(mds->mds_osc_exp, flags);
299 /* if obd_disconnect fails (probably because the
300 * export was disconnected by class_disconnect_exports)
301 * then we just need to drop our ref. */
303 class_export_put(mds->mds_osc_exp);
304 mds->mds_osc_exp = NULL;
305 mds->mds_osc_obd = NULL;
311 int mds_iocontrol(unsigned int cmd, struct obd_export *exp, int len,
312 void *karg, void *uarg)
314 static struct obd_uuid cfg_uuid = { .uuid = "config_uuid" };
315 struct obd_device *obd = exp->exp_obd;
316 struct mds_obd *mds = &obd->u.mds;
317 struct obd_ioctl_data *data = karg;
318 struct lvfs_run_ctxt saved;
322 case OBD_IOC_RECORD: {
323 char *name = data->ioc_inlbuf1;
324 if (mds->mds_cfg_llh)
327 push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
328 rc = llog_create(llog_get_context(&obd->obd_llogs,
329 LLOG_CONFIG_ORIG_CTXT),
330 &mds->mds_cfg_llh, NULL, name);
332 llog_init_handle(mds->mds_cfg_llh, LLOG_F_IS_PLAIN,
335 mds->mds_cfg_llh = NULL;
336 pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
341 case OBD_IOC_ENDRECORD: {
342 if (!mds->mds_cfg_llh)
345 push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
346 rc = llog_close(mds->mds_cfg_llh);
347 pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
349 mds->mds_cfg_llh = NULL;
353 case OBD_IOC_CLEAR_LOG: {
354 char *name = data->ioc_inlbuf1;
355 if (mds->mds_cfg_llh)
358 push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
359 rc = llog_create(llog_get_context(&obd->obd_llogs,
360 LLOG_CONFIG_ORIG_CTXT),
361 &mds->mds_cfg_llh, NULL, name);
363 llog_init_handle(mds->mds_cfg_llh, LLOG_F_IS_PLAIN,
366 rc = llog_destroy(mds->mds_cfg_llh);
367 llog_free_handle(mds->mds_cfg_llh);
369 pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
371 mds->mds_cfg_llh = NULL;
375 case OBD_IOC_DORECORD: {
377 struct llog_rec_hdr rec;
378 if (!mds->mds_cfg_llh)
381 rec.lrh_len = llog_data_len(data->ioc_plen1);
383 if (data->ioc_type == LUSTRE_CFG_TYPE) {
384 rec.lrh_type = OBD_CFG_REC;
385 } else if (data->ioc_type == PORTALS_CFG_TYPE) {
386 rec.lrh_type = PTL_CFG_REC;
388 CERROR("unknown cfg record type:%d \n", data->ioc_type);
392 OBD_ALLOC(cfg_buf, data->ioc_plen1);
395 rc = copy_from_user(cfg_buf, data->ioc_pbuf1, data->ioc_plen1);
397 OBD_FREE(cfg_buf, data->ioc_plen1);
401 push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
402 rc = llog_write_rec(mds->mds_cfg_llh, &rec, NULL, 0,
404 pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
406 OBD_FREE(cfg_buf, data->ioc_plen1);
410 case OBD_IOC_PARSE: {
411 struct llog_ctxt *ctxt =
412 llog_get_context(&obd->obd_llogs, LLOG_CONFIG_ORIG_CTXT);
413 push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
414 rc = class_config_parse_llog(ctxt, data->ioc_inlbuf1, NULL);
415 pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
422 case OBD_IOC_DUMP_LOG: {
423 struct llog_ctxt *ctxt =
424 llog_get_context(&obd->obd_llogs, LLOG_CONFIG_ORIG_CTXT);
425 push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
426 rc = class_config_dump_llog(ctxt, data->ioc_inlbuf1, NULL);
427 pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
434 case OBD_IOC_SET_READONLY: {
436 struct inode *inode = obd->u.mds.mds_sb->s_root->d_inode;
437 BDEVNAME_DECLARE_STORAGE(tmp);
438 CERROR("setting device %s read-only\n",
439 ll_bdevname(obd->u.mds.mds_sb, tmp));
441 handle = fsfilt_start(obd, inode, FSFILT_OP_MKNOD, NULL);
443 rc = fsfilt_commit(obd, obd->u.mds.mds_sb, inode, handle, 1);
445 dev_set_rdonly(ll_sbdev(obd->u.mds.mds_sb), 2);
449 case OBD_IOC_CATLOGLIST: {
450 int count = mds->mds_lov_desc.ld_tgt_count;
451 rc = llog_catalog_list(obd, count, data);
455 case OBD_IOC_LLOG_CHECK:
456 case OBD_IOC_LLOG_CANCEL:
457 case OBD_IOC_LLOG_REMOVE: {
458 struct llog_ctxt *ctxt =
459 llog_get_context(&obd->obd_llogs, LLOG_CONFIG_ORIG_CTXT);
460 char name[32] = "CATLIST";
463 obd_llog_finish(obd, &obd->obd_llogs,
464 mds->mds_lov_desc.ld_tgt_count);
465 push_ctxt(&saved, ctxt->loc_lvfs_ctxt, NULL);
466 rc = llog_ioctl(ctxt, cmd, data);
467 pop_ctxt(&saved, ctxt->loc_lvfs_ctxt, NULL);
468 obd_llog_cat_initialize(obd, &obd->obd_llogs,
469 mds->mds_lov_desc.ld_tgt_count,
471 rc2 = obd_set_info(mds->mds_osc_exp, strlen("mds_conn"),
472 "mds_conn", 0, NULL);
477 case OBD_IOC_LLOG_INFO:
478 case OBD_IOC_LLOG_PRINT: {
479 struct llog_ctxt *ctxt =
480 llog_get_context(&obd->obd_llogs, LLOG_CONFIG_ORIG_CTXT);
482 push_ctxt(&saved, ctxt->loc_lvfs_ctxt, NULL);
483 rc = llog_ioctl(ctxt, cmd, data);
484 pop_ctxt(&saved, ctxt->loc_lvfs_ctxt, NULL);
489 case OBD_IOC_ABORT_RECOVERY:
490 CERROR("aborting recovery for device %s\n", obd->obd_name);
491 target_abort_recovery(obd);
501 struct mds_lov_sync_info {
502 struct obd_device *mlsi_obd; /* the lov device to sync */
503 struct obd_uuid *mlsi_uuid; /* target to sync */
506 int mds_lov_synchronize(void *data)
508 struct mds_lov_sync_info *mlsi = data;
509 struct llog_ctxt *ctxt;
510 struct obd_device *obd;
511 struct obd_uuid *uuid;
518 SIGNAL_MASK_LOCK(current, flags);
519 sigfillset(¤t->blocked);
521 SIGNAL_MASK_UNLOCK(current, flags);
523 obd = mlsi->mlsi_obd;
524 uuid = mlsi->mlsi_uuid;
526 OBD_FREE(mlsi, sizeof(*mlsi));
528 LASSERT(obd != NULL);
529 LASSERT(uuid != NULL);
531 rc = obd_set_info(obd->u.mds.mds_osc_exp, strlen("mds_conn"),
532 "mds_conn", 0, uuid);
536 ctxt = llog_get_context(&obd->obd_llogs, LLOG_UNLINK_ORIG_CTXT);
537 LASSERT(ctxt != NULL);
539 rc = llog_connect(ctxt, obd->u.mds.mds_lov_desc.ld_tgt_count,
542 CERROR("%s: failed at llog_origin_connect: %d\n",
547 CWARN("MDS %s: %s now active, resetting orphans\n",
548 obd->obd_name, uuid->uuid);
549 rc = mds_lov_clearorphans(&obd->u.mds, uuid);
551 CERROR("%s: failed at mds_lov_clearorphans: %d\n",
559 int mds_lov_start_synchronize(struct obd_device *obd, struct obd_uuid *uuid)
561 struct mds_lov_sync_info *mlsi;
566 OBD_ALLOC(mlsi, sizeof(*mlsi));
570 mlsi->mlsi_obd = obd;
571 mlsi->mlsi_uuid = uuid;
573 rc = kernel_thread(mds_lov_synchronize, mlsi, CLONE_VM | CLONE_FILES);
575 CERROR("%s: error starting mds_lov_synchronize: %d\n",
578 CDEBUG(D_HA, "%s: mds_lov_synchronize thread: %d\n",
586 int mds_notify(struct obd_device *obd, struct obd_device *watched, int active)
588 struct obd_uuid *uuid;
595 if (strcmp(watched->obd_type->typ_name, LUSTRE_OSC_NAME)) {
596 CERROR("unexpected notification of %s %s!\n",
597 watched->obd_type->typ_name, watched->obd_name);
601 uuid = &watched->u.cli.cl_import->imp_target_uuid;
602 if (obd->obd_recovering) {
603 CWARN("MDS %s: in recovery, not resetting orphans on %s\n",
604 obd->obd_name, uuid->uuid);
606 rc = mds_lov_start_synchronize(obd, uuid);
611 /* Convert the on-disk LOV EA structre.
612 * We always try to convert from an old LOV EA format to the common in-memory
613 * (lsm) format (obd_unpackmd() understands the old on-disk (lmm) format) and
614 * then convert back to the new on-disk format and save it back to disk
615 * (obd_packmd() only ever saves to the new on-disk format) so we don't have
616 * to convert it each time this inode is accessed.
618 * This function is a bit interesting in the error handling. We can safely
619 * ship the old lmm to the client in case of failure, since it uses the same
620 * obd_unpackmd() code and can do the conversion if the MDS fails for some
621 * reason. We will not delete the old lmm data until we have written the
622 * new format lmm data in fsfilt_set_md(). */
623 int mds_convert_lov_ea(struct obd_device *obd, struct inode *inode,
624 struct lov_mds_md *lmm, int lmm_size)
626 struct lov_stripe_md *lsm = NULL;
631 if (le32_to_cpu(lmm->lmm_magic) == LOV_MAGIC)
634 CWARN("converting LOV EA on %lu/%u from V0 to V1\n",
635 inode->i_ino, inode->i_generation);
636 rc = obd_unpackmd(obd->u.mds.mds_osc_exp, &lsm, lmm, lmm_size);
640 rc = obd_packmd(obd->u.mds.mds_osc_exp, &lmm, lsm);
645 handle = fsfilt_start(obd, inode, FSFILT_OP_SETATTR, NULL);
646 if (IS_ERR(handle)) {
647 rc = PTR_ERR(handle);
651 rc = fsfilt_set_md(obd, inode, handle, lmm, lmm_size);
653 err = fsfilt_commit(obd, obd->u.mds.mds_sb, inode, handle, 0);
655 rc = err ? err : lmm_size;
658 obd_free_memmd(obd->u.mds.mds_osc_exp, &lsm);