1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
5 * Lustre Metadata Server (mds) handling of striped file data
7 * Copyright (C) 2001-2003 Cluster File Systems, Inc.
9 * This file is part of Lustre, http://www.lustre.org.
11 * Lustre is free software; you can redistribute it and/or
12 * modify it under the terms of version 2 of the GNU General Public
13 * License as published by the Free Software Foundation.
15 * Lustre is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
20 * You should have received a copy of the GNU General Public License
21 * along with Lustre; if not, write to the Free Software
22 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26 # define EXPORT_SYMTAB
28 #define DEBUG_SUBSYSTEM S_MDS
30 #include <linux/module.h>
31 #include <linux/lustre_mds.h>
32 #include <linux/lustre_idl.h>
33 #include <linux/obd_class.h>
34 #include <linux/obd_lov.h>
35 #include <linux/lustre_lib.h>
36 #include <linux/lustre_fsfilt.h>
38 #include "mds_internal.h"
43 * - magic in mea struct
44 * - error handling is totally missed
47 int mds_lmv_connect(struct obd_device *obd, char * lmv_name)
49 struct mds_obd *mds = &obd->u.mds;
50 struct lustre_handle conn = {0,};
55 if (IS_ERR(mds->mds_lmv_obd))
56 RETURN(PTR_ERR(mds->mds_lmv_obd));
61 mds->mds_lmv_obd = class_name2obd(lmv_name);
62 if (!mds->mds_lmv_obd) {
63 CERROR("MDS cannot locate LMV %s\n",
65 mds->mds_lmv_obd = ERR_PTR(-ENOTCONN);
69 rc = obd_connect(&conn, mds->mds_lmv_obd, &obd->obd_uuid);
71 CERROR("MDS cannot connect to LMV %s (%d)\n",
73 mds->mds_lmv_obd = ERR_PTR(rc);
76 mds->mds_lmv_exp = class_conn2export(&conn);
77 if (mds->mds_lmv_exp == NULL)
78 CERROR("can't get export!\n");
80 rc = obd_register_observer(mds->mds_lmv_obd, obd);
82 CERROR("MDS cannot register as observer of LMV %s (%d)\n",
87 /* retrieve size of EA */
88 rc = obd_get_info(mds->mds_lmv_exp, strlen("mdsize"), "mdsize",
92 if (mdsize > mds->mds_max_mdsize)
93 mds->mds_max_mdsize = mdsize;
95 /* find our number in LMV cluster */
96 rc = obd_get_info(mds->mds_lmv_exp, strlen("mdsnum"), "mdsnum",
100 mds->mds_num = mdsize;
102 rc = obd_set_info(mds->mds_lmv_exp, strlen("inter_mds"),
103 "inter_mds", 0, NULL);
112 /* FIXME: cleanups here! */
113 obd_disconnect(mds->mds_lmv_exp, 0);
114 mds->mds_lmv_exp = NULL;
115 mds->mds_lmv_obd = ERR_PTR(rc);
119 int mds_lmv_postsetup(struct obd_device *obd)
121 struct mds_obd *mds = &obd->u.mds;
123 if (mds->mds_lmv_exp)
124 obd_init_ea_size(mds->mds_lmv_exp, mds->mds_max_mdsize,
125 mds->mds_max_cookiesize);
129 int mds_lmv_disconnect(struct obd_device *obd, int flags)
131 struct mds_obd *mds = &obd->u.mds;
135 if (!IS_ERR(mds->mds_lmv_obd) && mds->mds_lmv_exp != NULL) {
137 obd_register_observer(mds->mds_lmv_obd, NULL);
139 rc = obd_disconnect(mds->mds_lmv_exp, flags);
140 /* if obd_disconnect fails (probably because the
141 * export was disconnected by class_disconnect_exports)
142 * then we just need to drop our ref. */
144 class_export_put(mds->mds_lmv_exp);
145 mds->mds_lmv_exp = NULL;
146 mds->mds_lmv_obd = NULL;
153 int mds_get_lmv_attr(struct obd_device *obd, struct inode *inode,
154 struct mea **mea, int *mea_size)
156 struct mds_obd *mds = &obd->u.mds;
160 if (!mds->mds_lmv_obd)
163 /* first calculate mea size */
164 *mea_size = obd_alloc_diskmd(mds->mds_lmv_exp,
165 (struct lov_mds_md **) mea);
166 /* FIXME: error handling here */
167 LASSERT(*mea != NULL);
170 rc = fsfilt_get_md(obd, inode, *mea, *mea_size);
172 /* FIXME: error handling here */
174 OBD_FREE(*mea, *mea_size);
191 #define DIR_ROUND (DIR_PAD - 1)
192 #define DIR_REC_LEN(name_len) (((name_len) + 12 + DIR_ROUND) & ~DIR_ROUND)
194 /* this struct holds dir entries for particular MDS to be flushed */
196 struct list_head list;
201 struct brw_page brwc;
204 struct dirsplit_control {
205 struct obd_device *obd;
207 struct dentry *dentry;
209 struct dir_cache *cache;
212 static int dc_new_page_to_cache(struct dir_cache * dirc)
216 if (!list_empty(&dirc->list) && dirc->free > sizeof(__u16)) {
217 /* current page became full, mark the end */
218 struct dir_entry *de = dirc->cur;
222 page = alloc_page(GFP_KERNEL);
225 list_add_tail(&page->list, &dirc->list);
226 dirc->cur = page_address(page);
227 dirc->free = PAGE_SIZE;
231 static int retrieve_generation_numbers(struct dirsplit_control *dc, void *buf)
233 struct mds_obd *mds = &dc->obd->u.mds;
234 struct dir_entry *de;
235 struct dentry *dentry;
238 end = buf + PAGE_SIZE;
239 de = (struct dir_entry *) buf;
240 while ((char *) de < end && de->namelen) {
241 /* lookup an inode */
242 LASSERT(de->namelen <= 255);
243 dentry = ll_lookup_one_len(de->name, dc->dentry, de->namelen);
244 if (IS_ERR(dentry)) {
245 CERROR("can't lookup %*s: %d\n", de->namelen,
246 de->name, (int) PTR_ERR(dentry));
249 if (dentry->d_inode != NULL) {
250 de->mds = mds->mds_num;
251 de->ino = dentry->d_inode->i_ino;
252 de->generation = dentry->d_inode->i_generation;
253 } else if (dentry->d_flags & DCACHE_CROSS_REF) {
254 de->mds = dentry->d_mdsnum;
255 de->ino = dentry->d_inum;
256 de->generation = dentry->d_generation;
258 CERROR("can't lookup %*s\n", de->namelen, de->name);
264 de = (struct dir_entry *)
265 ((char *) de + DIR_REC_LEN(de->namelen));
270 static int flush_buffer_onto_mds(struct dirsplit_control *dc, int mdsnum)
272 struct mds_obd *mds = &dc->obd->u.mds;
273 struct dir_cache *ca;
274 struct list_head *cur, *tmp;
276 ca = dc->cache + mdsnum;
278 if (ca->free > sizeof(__u16)) {
279 /* current page became full, mark the end */
280 struct dir_entry *de = ca->cur;
284 list_for_each_safe(cur, tmp, &ca->list) {
287 page = list_entry(cur, struct page, list);
288 LASSERT(page != NULL);
290 retrieve_generation_numbers(dc, page_address(page));
294 ca->brwc.count = PAGE_SIZE;
296 ca->oa.o_mds = mdsnum;
297 obd_brw(OBD_BRW_WRITE, mds->mds_lmv_exp, &ca->oa,
298 (struct lov_stripe_md *) dc->mea,
301 list_del(&page->list);
307 static int filldir(void * __buf, const char * name, int namlen, loff_t offset,
308 ino_t ino, unsigned int d_type)
310 struct dirsplit_control *dc = __buf;
311 struct mds_obd *mds = &dc->obd->u.mds;
312 struct dir_cache *ca;
313 struct dir_entry *de;
318 if (name[0] == '.' && (namlen == 1 ||
319 (namlen == 2 && name[1] == '.'))) {
320 /* skip special entries */
325 newmds = mea_name2idx(dc->mea, (char *) name, namlen);
327 if (newmds == mds->mds_num) {
328 /* this entry remains on the current MDS, skip moving */
332 OBD_ALLOC(n, namlen + 1);
333 memcpy(n, name, namlen);
334 n[namlen] = (char) 0;
336 OBD_FREE(n, namlen + 1);
338 /* check for space in buffer for new entry */
339 ca = dc->cache + newmds;
340 if (DIR_REC_LEN(namlen) > ca->free) {
341 int err = dc_new_page_to_cache(ca);
345 /* insert found entry into buffer to be flushed later */
346 /* NOTE: we'll fill generations number later, because we
347 * it's stored in inode, thus we need to lookup an entry,
348 * but directory is locked for readdir(), so we delay this */
352 de->namelen = namlen;
353 memcpy(de->name, name, namlen);
354 ca->cur += DIR_REC_LEN(namlen);
355 ca->free -= DIR_REC_LEN(namlen);
361 int scan_and_distribute(struct obd_device *obd, struct dentry *dentry,
364 struct inode *dir = dentry->d_inode;
365 struct dirsplit_control dc;
370 nlen = strlen("__iopen__/") + 10 + 1;
371 OBD_ALLOC(file_name, nlen);
374 i = sprintf(file_name, "__iopen__/0x%lx", dentry->d_inode->i_ino);
376 file = filp_open(file_name, O_RDONLY, 0);
378 CERROR("can't open directory %s: %d\n",
379 file_name, (int) PTR_ERR(file));
380 OBD_FREE(file_name, nlen);
381 RETURN(PTR_ERR(file));
384 memset(&dc, 0, sizeof(dc));
389 OBD_ALLOC(dc.cache, sizeof(struct dir_cache) * mea->mea_count);
390 LASSERT(dc.cache != NULL);
391 for (i = 0; i < mea->mea_count; i++) {
392 INIT_LIST_HEAD(&dc.cache[i].list);
393 dc.cache[i].free = 0;
394 dc.cache[i].cached = 0;
397 err = vfs_readdir(file, filldir, &dc);
401 for (i = 0; i < mea->mea_count; i++) {
402 if (dc.cache[i].cached)
403 flush_buffer_onto_mds(&dc, i);
406 OBD_FREE(dc.cache, sizeof(struct dir_cache) * mea->mea_count);
407 OBD_FREE(file_name, nlen);
412 #define MAX_DIR_SIZE (64 * 1024)
415 * must not be called on already splitted directories
417 int mds_try_to_split_dir(struct obd_device *obd,
418 struct dentry *dentry, struct mea **mea, int nstripes)
420 struct inode *dir = dentry->d_inode;
421 struct mds_obd *mds = &obd->u.mds;
422 struct mea *tmea = NULL;
423 struct obdo *oa = NULL;
424 int rc, mea_size = 0;
429 if (!mds->mds_lmv_obd)
432 /* don't split root directory */
433 if (dentry->d_inode->i_ino == mds->mds_rootfid.id)
436 /* we want to split only large dirs. this may be already
437 * splitted dir or a slave dir created during splitting */
438 if (dir->i_size < MAX_DIR_SIZE)
441 /* check is directory marked non-splittable */
445 CDEBUG(D_OTHER, "%s: split directory %u/%lu/%lu\n",
446 obd->obd_name, mds->mds_num, dir->i_ino,
447 (unsigned long) dir->i_generation);
451 mea_size = obd_size_diskmd(mds->mds_lmv_exp, NULL);
453 /* FIXME: Actually we may only want to allocate enough space for
454 * necessary amount of stripes, but on the other hand with this
455 * approach of allocating maximal possible amount of MDS slots,
456 * it would be easier to split the dir over more MDSes */
457 rc = obd_alloc_diskmd(mds->mds_lmv_exp, (void *) mea);
460 (*mea)->mea_count = nstripes;
462 #warning "we have to take EX lock on a dir for splitting"
464 /* 1) create directory objects on slave MDS'es */
465 /* FIXME: should this be OBD method? */
467 /* FIXME: error handling here */
469 oa->o_id = dir->i_ino;
470 oa->o_generation = dir->i_generation;
471 obdo_from_inode(oa, dir, OBD_MD_FLTYPE | OBD_MD_FLATIME |
472 OBD_MD_FLMTIME | OBD_MD_FLCTIME |
473 OBD_MD_FLUID | OBD_MD_FLGID);
474 oa->o_gr = FILTER_GROUP_FIRST_MDS + mds->mds_num;
475 oa->o_valid |= OBD_MD_FLID | OBD_MD_FLFLAGS | OBD_MD_FLGROUP;
476 oa->o_mode = dir->i_mode;
477 CDEBUG(D_OTHER, "%s: create subdirs with mode %o, uid %u, gid %u\n",
478 obd->obd_name, dir->i_mode, dir->i_uid, dir->i_gid);
480 rc = obd_create(mds->mds_lmv_exp, oa,
481 (struct lov_stripe_md **) mea, NULL);
482 /* FIXME: error handling here */
484 CDEBUG(D_OTHER, "%d dirobjects created\n",
485 (int) (*mea)->mea_count);
487 /* 2) update dir attribute */
489 handle = fsfilt_start(obd, dir, FSFILT_OP_SETATTR, NULL);
490 LASSERT(!IS_ERR(handle));
491 rc = fsfilt_set_md(obd, dir, handle, *mea, mea_size);
493 fsfilt_commit(obd, dir, handle, 0);
498 /* 3) read through the dir and distribute it over objects */
499 scan_and_distribute(obd, dentry, *mea);
502 obd_free_diskmd(mds->mds_lmv_exp,
503 (struct lov_mds_md **) mea);
507 static int filter_start_page_write(struct inode *inode,
508 struct niobuf_local *lnb)
510 struct page *page = alloc_pages(GFP_HIGHUSER, 0);
512 CERROR("no memory for a temp page\n");
513 RETURN(lnb->rc = -ENOMEM);
515 POISON_PAGE(page, 0xf1);
516 page->index = lnb->offset >> PAGE_SHIFT;
522 struct dentry *filter_fid2dentry(struct obd_device *obd,
523 struct dentry *dir_dentry,
524 obd_gr group, obd_id id);
526 int mds_preprw(int cmd, struct obd_export *exp, struct obdo *oa,
527 int objcount, struct obd_ioobj *obj,
528 int niocount, struct niobuf_remote *nb,
529 struct niobuf_local *res,
530 struct obd_trans_info *oti)
532 struct mds_obd *mds = &exp->exp_obd->u.mds;
533 struct niobuf_remote *rnb;
534 struct niobuf_local *lnb = NULL;
535 int rc = 0, i, tot_bytes = 0;
536 unsigned long now = jiffies;
537 struct dentry *dentry;
540 LASSERT(objcount == 1);
541 LASSERT(obj->ioo_bufcnt > 0);
543 memset(res, 0, niocount * sizeof(*res));
545 fid.id = obj->ioo_id;
546 fid.generation = obj->ioo_gr;
547 dentry = mds_fid2dentry(mds, &fid, NULL);
548 LASSERT(!IS_ERR(dentry));
550 if (dentry->d_inode == NULL) {
551 CERROR("trying to BRW to non-existent file "LPU64"\n",
554 GOTO(cleanup, rc = -ENOENT);
557 if (time_after(jiffies, now + 15 * HZ))
558 CERROR("slow preprw_write setup %lus\n", (jiffies - now) / HZ);
560 CDEBUG(D_INFO, "preprw_write setup: %lu jiffies\n",
563 for (i = 0, rnb = nb, lnb = res; i < obj->ioo_bufcnt;
565 lnb->dentry = dentry;
566 lnb->offset = rnb->offset;
568 lnb->flags = rnb->flags;
570 rc = filter_start_page_write(dentry->d_inode, lnb);
572 CDEBUG(rc == -ENOSPC ? D_INODE : D_ERROR, "page err %u@"
573 LPU64" %u/%u %p: rc %d\n", lnb->len, lnb->offset,
574 i, obj->ioo_bufcnt, dentry, rc);
576 __free_pages(lnb->page, 0);
580 tot_bytes += lnb->len;
583 if (time_after(jiffies, now + 15 * HZ))
584 CERROR("slow start_page_write %lus\n", (jiffies - now) / HZ);
586 CDEBUG(D_INFO, "start_page_write: %lu jiffies\n",
594 int mds_commitrw(int cmd, struct obd_export *exp, struct obdo *oa,
595 int objcount, struct obd_ioobj *obj, int niocount,
596 struct niobuf_local *res, struct obd_trans_info *oti,
599 struct obd_device *obd = exp->exp_obd;
600 struct niobuf_local *lnb;
601 struct inode *inode = NULL;
602 int rc = 0, i, cleanup_phase = 0, err, entries = 0;
605 LASSERT(objcount == 1);
606 LASSERT(current->journal_info == NULL);
609 inode = res->dentry->d_inode;
611 for (i = 0, lnb = res; i < obj->ioo_bufcnt; i++, lnb++) {
613 struct dir_entry *de;
615 buf = kmap(lnb->page);
616 LASSERT(buf != NULL);
617 end = buf + lnb->len;
618 de = (struct dir_entry *) buf;
619 while ((char *) de < end && de->namelen) {
620 err = fsfilt_add_dir_entry(obd, res->dentry, de->name,
621 de->namelen, de->ino,
622 de->generation, de->mds);
623 /* FIXME: remove entries from the original dir */
624 #warning "removing entries from the original dir"
626 de = (struct dir_entry *)
627 ((char *) de + DIR_REC_LEN(de->namelen));
633 for (i = 0, lnb = res; i < obj->ioo_bufcnt; i++, lnb++)
634 __free_page(lnb->page);
640 int mds_choose_mdsnum(struct obd_device *obd, const char *name, int len)
642 struct mds_obd *mds = &obd->u.mds;
643 struct lmv_obd *lmv = &mds->mds_lmv_exp->exp_obd->u.lmv;
646 i = raw_name2idx(lmv->count, name, len);