1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
5 * Lustre Metadata Server (mds) handling of striped file data
7 * Copyright (C) 2001-2003 Cluster File Systems, Inc.
9 * This file is part of Lustre, http://www.lustre.org.
11 * Lustre is free software; you can redistribute it and/or
12 * modify it under the terms of version 2 of the GNU General Public
13 * License as published by the Free Software Foundation.
15 * Lustre is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
20 * You should have received a copy of the GNU General Public License
21 * along with Lustre; if not, write to the Free Software
22 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26 # define EXPORT_SYMTAB
28 #define DEBUG_SUBSYSTEM S_MDS
30 #include <linux/module.h>
31 #include <linux/lustre_mds.h>
32 #include <linux/lustre_idl.h>
33 #include <linux/obd_class.h>
34 #include <linux/obd_lov.h>
35 #include <linux/lustre_lib.h>
36 #include <linux/lustre_fsfilt.h>
38 #include "mds_internal.h"
43 * - magic in mea struct
44 * - error handling is totally missed
47 int mds_lmv_connect(struct obd_device *obd, char * lmv_name)
49 struct mds_obd *mds = &obd->u.mds;
50 struct lustre_handle conn = {0,};
55 if (IS_ERR(mds->mds_lmv_obd))
56 RETURN(PTR_ERR(mds->mds_lmv_obd));
61 mds->mds_lmv_obd = class_name2obd(lmv_name);
62 if (!mds->mds_lmv_obd) {
63 CERROR("MDS cannot locate LMV %s\n",
65 mds->mds_lmv_obd = ERR_PTR(-ENOTCONN);
69 rc = obd_connect(&conn, mds->mds_lmv_obd, &obd->obd_uuid);
71 CERROR("MDS cannot connect to LMV %s (%d)\n",
73 mds->mds_lmv_obd = ERR_PTR(rc);
76 mds->mds_lmv_exp = class_conn2export(&conn);
77 if (mds->mds_lmv_exp == NULL)
78 CERROR("can't get export!\n");
80 rc = obd_register_observer(mds->mds_lmv_obd, obd);
82 CERROR("MDS cannot register as observer of LMV %s (%d)\n",
87 /* retrieve size of EA */
88 rc = obd_get_info(mds->mds_lmv_exp, strlen("mdsize"), "mdsize",
92 if (mdsize > mds->mds_max_mdsize)
93 mds->mds_max_mdsize = mdsize;
95 /* find our number in LMV cluster */
96 rc = obd_get_info(mds->mds_lmv_exp, strlen("mdsnum"), "mdsnum",
100 mds->mds_num = mdsize;
102 rc = obd_set_info(mds->mds_lmv_exp, strlen("inter_mds"),
103 "inter_mds", 0, NULL);
112 /* FIXME: cleanups here! */
113 obd_disconnect(mds->mds_lmv_exp, 0);
114 mds->mds_lmv_exp = NULL;
115 mds->mds_lmv_obd = ERR_PTR(rc);
119 int mds_lmv_postsetup(struct obd_device *obd)
121 struct mds_obd *mds = &obd->u.mds;
123 if (mds->mds_lmv_exp)
124 obd_init_ea_size(mds->mds_lmv_exp, mds->mds_max_mdsize,
125 mds->mds_max_cookiesize);
129 int mds_lmv_disconnect(struct obd_device *obd, int flags)
131 struct mds_obd *mds = &obd->u.mds;
135 if (!IS_ERR(mds->mds_lmv_obd) && mds->mds_lmv_exp != NULL) {
137 obd_register_observer(mds->mds_lmv_obd, NULL);
139 rc = obd_disconnect(mds->mds_lmv_exp, flags);
140 /* if obd_disconnect fails (probably because the
141 * export was disconnected by class_disconnect_exports)
142 * then we just need to drop our ref. */
144 class_export_put(mds->mds_lmv_exp);
145 mds->mds_lmv_exp = NULL;
146 mds->mds_lmv_obd = NULL;
153 int mds_get_lmv_attr(struct obd_device *obd, struct inode *inode,
154 struct mea **mea, int *mea_size)
156 struct mds_obd *mds = &obd->u.mds;
160 if (!mds->mds_lmv_obd)
163 /* first calculate mea size */
164 *mea_size = obd_alloc_diskmd(mds->mds_lmv_exp,
165 (struct lov_mds_md **)mea);
166 /* FIXME: error handling here */
167 LASSERT(*mea != NULL);
170 rc = fsfilt_get_md(obd, inode, *mea, *mea_size);
172 /* FIXME: error handling here */
174 OBD_FREE(*mea, *mea_size);
191 #define DIR_ROUND (DIR_PAD - 1)
192 #define DIR_REC_LEN(name_len) (((name_len) + 12 + DIR_ROUND) & ~DIR_ROUND)
194 /* this struct holds dir entries for particular MDS to be flushed */
196 struct list_head list;
201 struct brw_page brwc;
204 struct dirsplit_control {
205 struct obd_device *obd;
207 struct dentry *dentry;
209 struct dir_cache *cache;
212 static int dc_new_page_to_cache(struct dir_cache * dirc)
216 if (!list_empty(&dirc->list) && dirc->free > sizeof(__u16)) {
217 /* current page became full, mark the end */
218 struct dir_entry *de = dirc->cur;
222 page = alloc_page(GFP_KERNEL);
225 list_add_tail(&page->list, &dirc->list);
226 dirc->cur = page_address(page);
227 dirc->free = PAGE_SIZE;
231 static int retrieve_generation_numbers(struct dirsplit_control *dc, void *buf)
233 struct mds_obd *mds = &dc->obd->u.mds;
234 struct dir_entry *de;
235 struct dentry *dentry;
238 end = buf + PAGE_SIZE;
239 de = (struct dir_entry *) buf;
240 while ((char *) de < end && de->namelen) {
241 /* lookup an inode */
242 LASSERT(de->namelen <= 255);
243 dentry = ll_lookup_one_len(de->name, dc->dentry, de->namelen);
244 if (IS_ERR(dentry)) {
245 CERROR("can't lookup %*s: %d\n", de->namelen,
246 de->name, (int) PTR_ERR(dentry));
249 if (dentry->d_inode != NULL) {
250 de->mds = mds->mds_num;
251 de->ino = dentry->d_inode->i_ino;
252 de->generation = dentry->d_inode->i_generation;
253 } else if (dentry->d_flags & DCACHE_CROSS_REF) {
254 de->mds = dentry->d_mdsnum;
255 de->ino = dentry->d_inum;
256 de->generation = dentry->d_generation;
258 CERROR("can't lookup %*s\n", de->namelen, de->name);
264 de = (struct dir_entry *)
265 ((char *) de + DIR_REC_LEN(de->namelen));
270 static int flush_buffer_onto_mds(struct dirsplit_control *dc, int mdsnum)
272 struct mds_obd *mds = &dc->obd->u.mds;
273 struct dir_cache *ca;
274 struct list_head *cur, *tmp;
276 ca = dc->cache + mdsnum;
278 if (ca->free > sizeof(__u16)) {
279 /* current page became full, mark the end */
280 struct dir_entry *de = ca->cur;
284 list_for_each_safe(cur, tmp, &ca->list) {
287 page = list_entry(cur, struct page, list);
288 LASSERT(page != NULL);
290 retrieve_generation_numbers(dc, page_address(page));
294 ca->brwc.count = PAGE_SIZE;
296 ca->oa.o_mds = mdsnum;
297 obd_brw(OBD_BRW_WRITE, mds->mds_lmv_exp, &ca->oa,
298 (struct lov_stripe_md *) dc->mea,
301 list_del(&page->list);
307 static int filldir(void * __buf, const char * name, int namlen, loff_t offset,
308 ino_t ino, unsigned int d_type)
310 struct dirsplit_control *dc = __buf;
311 struct mds_obd *mds = &dc->obd->u.mds;
312 struct dir_cache *ca;
313 struct dir_entry *de;
318 if (name[0] == '.' && (namlen == 1 ||
319 (namlen == 2 && name[1] == '.'))) {
320 /* skip special entries */
325 newmds = mea_name2idx(dc->mea, (char *) name, namlen);
327 if (newmds == mds->mds_num) {
328 /* this entry remains on the current MDS, skip moving */
332 OBD_ALLOC(n, namlen + 1);
333 memcpy(n, name, namlen);
334 n[namlen] = (char) 0;
336 OBD_FREE(n, namlen + 1);
338 /* check for space in buffer for new entry */
339 ca = dc->cache + newmds;
340 if (DIR_REC_LEN(namlen) > ca->free) {
341 int err = dc_new_page_to_cache(ca);
345 /* insert found entry into buffer to be flushed later */
346 /* NOTE: we'll fill generations number later, because we
347 * it's stored in inode, thus we need to lookup an entry,
348 * but directory is locked for readdir(), so we delay this */
352 de->namelen = namlen;
353 memcpy(de->name, name, namlen);
354 ca->cur += DIR_REC_LEN(namlen);
355 ca->free -= DIR_REC_LEN(namlen);
361 int scan_and_distribute(struct obd_device *obd, struct dentry *dentry,
364 struct inode *dir = dentry->d_inode;
365 struct dirsplit_control dc;
370 nlen = strlen("__iopen__/") + 10 + 1;
371 OBD_ALLOC(file_name, nlen);
374 i = sprintf(file_name, "__iopen__/0x%lx", dentry->d_inode->i_ino);
376 file = filp_open(file_name, O_RDONLY, 0);
378 CERROR("can't open directory %s: %d\n",
379 file_name, (int) PTR_ERR(file));
380 OBD_FREE(file_name, nlen);
381 RETURN(PTR_ERR(file));
384 memset(&dc, 0, sizeof(dc));
389 OBD_ALLOC(dc.cache, sizeof(struct dir_cache) * mea->mea_count);
390 LASSERT(dc.cache != NULL);
391 for (i = 0; i < mea->mea_count; i++) {
392 INIT_LIST_HEAD(&dc.cache[i].list);
393 dc.cache[i].free = 0;
394 dc.cache[i].cached = 0;
397 err = vfs_readdir(file, filldir, &dc);
401 for (i = 0; i < mea->mea_count; i++) {
402 if (dc.cache[i].cached)
403 flush_buffer_onto_mds(&dc, i);
406 OBD_FREE(dc.cache, sizeof(struct dir_cache) * mea->mea_count);
407 OBD_FREE(file_name, nlen);
412 #define MAX_DIR_SIZE (64 * 1024)
414 int mds_splitting_expected(struct obd_device *obd, struct dentry *dentry)
416 struct mds_obd *mds = &obd->u.mds;
417 struct mea *mea = NULL;
421 if (!mds->mds_lmv_obd)
425 if (dentry->d_inode == NULL)
428 /* a dir can be splitted only */
429 if (!S_ISDIR(dentry->d_inode->i_mode))
432 /* large enough to be splitted? */
433 if (dentry->d_inode->i_size < MAX_DIR_SIZE)
436 /* don't split root directory */
437 if (dentry->d_inode->i_ino == mds->mds_rootfid.id)
440 mds_get_lmv_attr(obd, dentry->d_inode, &mea, &size);
442 /* already splitted or slave object: shouldn't be splitted */
445 /* may be splitted */
455 * must not be called on already splitted directories
457 int mds_try_to_split_dir(struct obd_device *obd,
458 struct dentry *dentry, struct mea **mea, int nstripes)
460 struct inode *dir = dentry->d_inode;
461 struct mds_obd *mds = &obd->u.mds;
462 struct mea *tmea = NULL;
463 struct obdo *oa = NULL;
464 int rc, mea_size = 0;
468 /* TODO: optimization possible - we already may have mea here */
469 if (!mds_splitting_expected(obd, dentry))
471 LASSERT(mea == NULL || *mea == NULL);
473 CDEBUG(D_OTHER, "%s: split directory %u/%lu/%lu\n",
474 obd->obd_name, mds->mds_num, dir->i_ino,
475 (unsigned long) dir->i_generation);
479 mea_size = obd_size_diskmd(mds->mds_lmv_exp, NULL);
481 /* FIXME: Actually we may only want to allocate enough space for
482 * necessary amount of stripes, but on the other hand with this
483 * approach of allocating maximal possible amount of MDS slots,
484 * it would be easier to split the dir over more MDSes */
485 rc = obd_alloc_diskmd(mds->mds_lmv_exp, (void *) mea);
488 (*mea)->mea_count = nstripes;
490 /* 1) create directory objects on slave MDS'es */
491 /* FIXME: should this be OBD method? */
493 /* FIXME: error handling here */
495 oa->o_id = dir->i_ino;
496 oa->o_generation = dir->i_generation;
497 obdo_from_inode(oa, dir, OBD_MD_FLTYPE | OBD_MD_FLATIME |
498 OBD_MD_FLMTIME | OBD_MD_FLCTIME |
499 OBD_MD_FLUID | OBD_MD_FLGID);
500 oa->o_gr = FILTER_GROUP_FIRST_MDS + mds->mds_num;
501 oa->o_valid |= OBD_MD_FLID | OBD_MD_FLFLAGS | OBD_MD_FLGROUP;
502 oa->o_mode = dir->i_mode;
503 CDEBUG(D_OTHER, "%s: create subdirs with mode %o, uid %u, gid %u\n",
504 obd->obd_name, dir->i_mode, dir->i_uid, dir->i_gid);
506 rc = obd_create(mds->mds_lmv_exp, oa,
507 (struct lov_stripe_md **) mea, NULL);
508 /* FIXME: error handling here */
510 CDEBUG(D_OTHER, "%d dirobjects created\n",
511 (int) (*mea)->mea_count);
513 /* 2) update dir attribute */
515 handle = fsfilt_start(obd, dir, FSFILT_OP_SETATTR, NULL);
516 LASSERT(!IS_ERR(handle));
517 rc = fsfilt_set_md(obd, dir, handle, *mea, mea_size);
519 fsfilt_commit(obd, dir, handle, 0);
524 /* 3) read through the dir and distribute it over objects */
525 scan_and_distribute(obd, dentry, *mea);
528 obd_free_diskmd(mds->mds_lmv_exp,
529 (struct lov_mds_md **) mea);
533 static int filter_start_page_write(struct inode *inode,
534 struct niobuf_local *lnb)
536 struct page *page = alloc_pages(GFP_HIGHUSER, 0);
538 CERROR("no memory for a temp page\n");
539 RETURN(lnb->rc = -ENOMEM);
541 POISON_PAGE(page, 0xf1);
542 page->index = lnb->offset >> PAGE_SHIFT;
548 struct dentry *filter_fid2dentry(struct obd_device *obd,
549 struct dentry *dir_dentry,
550 obd_gr group, obd_id id);
552 int mds_preprw(int cmd, struct obd_export *exp, struct obdo *oa,
553 int objcount, struct obd_ioobj *obj,
554 int niocount, struct niobuf_remote *nb,
555 struct niobuf_local *res,
556 struct obd_trans_info *oti)
558 struct mds_obd *mds = &exp->exp_obd->u.mds;
559 struct niobuf_remote *rnb;
560 struct niobuf_local *lnb = NULL;
561 int rc = 0, i, tot_bytes = 0;
562 unsigned long now = jiffies;
563 struct dentry *dentry;
566 LASSERT(objcount == 1);
567 LASSERT(obj->ioo_bufcnt > 0);
569 memset(res, 0, niocount * sizeof(*res));
571 fid.id = obj->ioo_id;
572 fid.generation = obj->ioo_gr;
573 dentry = mds_fid2dentry(mds, &fid, NULL);
574 LASSERT(!IS_ERR(dentry));
576 if (dentry->d_inode == NULL) {
577 CERROR("trying to BRW to non-existent file "LPU64"\n",
580 GOTO(cleanup, rc = -ENOENT);
583 if (time_after(jiffies, now + 15 * HZ))
584 CERROR("slow preprw_write setup %lus\n", (jiffies - now) / HZ);
586 CDEBUG(D_INFO, "preprw_write setup: %lu jiffies\n",
589 for (i = 0, rnb = nb, lnb = res; i < obj->ioo_bufcnt;
591 lnb->dentry = dentry;
592 lnb->offset = rnb->offset;
594 lnb->flags = rnb->flags;
596 rc = filter_start_page_write(dentry->d_inode, lnb);
598 CDEBUG(rc == -ENOSPC ? D_INODE : D_ERROR, "page err %u@"
599 LPU64" %u/%u %p: rc %d\n", lnb->len, lnb->offset,
600 i, obj->ioo_bufcnt, dentry, rc);
602 __free_pages(lnb->page, 0);
606 tot_bytes += lnb->len;
609 if (time_after(jiffies, now + 15 * HZ))
610 CERROR("slow start_page_write %lus\n", (jiffies - now) / HZ);
612 CDEBUG(D_INFO, "start_page_write: %lu jiffies\n",
620 int mds_commitrw(int cmd, struct obd_export *exp, struct obdo *oa,
621 int objcount, struct obd_ioobj *obj, int niocount,
622 struct niobuf_local *res, struct obd_trans_info *oti,
625 struct obd_device *obd = exp->exp_obd;
626 struct niobuf_local *lnb;
627 struct inode *inode = NULL;
628 int rc = 0, i, cleanup_phase = 0, err, entries = 0;
631 LASSERT(objcount == 1);
632 LASSERT(current->journal_info == NULL);
635 inode = res->dentry->d_inode;
637 for (i = 0, lnb = res; i < obj->ioo_bufcnt; i++, lnb++) {
639 struct dir_entry *de;
641 buf = kmap(lnb->page);
642 LASSERT(buf != NULL);
643 end = buf + lnb->len;
644 de = (struct dir_entry *) buf;
645 while ((char *) de < end && de->namelen) {
646 err = fsfilt_add_dir_entry(obd, res->dentry, de->name,
647 de->namelen, de->ino,
648 de->generation, de->mds);
649 /* FIXME: remove entries from the original dir */
650 #warning "removing entries from the original dir"
652 de = (struct dir_entry *)
653 ((char *) de + DIR_REC_LEN(de->namelen));
659 for (i = 0, lnb = res; i < obj->ioo_bufcnt; i++, lnb++)
660 __free_page(lnb->page);
666 int mds_choose_mdsnum(struct obd_device *obd, const char *name, int len, int flags)
669 struct mds_obd *mds = &obd->u.mds;
670 int i = mds->mds_num;
672 if (flags & REC_REINT_CREATE) {
674 } else if (mds->mds_lmv_exp) {
675 lmv = &mds->mds_lmv_exp->exp_obd->u.lmv;
676 i = raw_name2idx(lmv->count, name, len);