1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
5 * Lustre Metadata Server (mds) handling of striped file data
7 * Copyright (C) 2001-2003 Cluster File Systems, Inc.
9 * This file is part of Lustre, http://www.lustre.org.
11 * Lustre is free software; you can redistribute it and/or
12 * modify it under the terms of version 2 of the GNU General Public
13 * License as published by the Free Software Foundation.
15 * Lustre is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
20 * You should have received a copy of the GNU General Public License
21 * along with Lustre; if not, write to the Free Software
22 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26 # define EXPORT_SYMTAB
28 #define DEBUG_SUBSYSTEM S_MDS
30 #include <linux/module.h>
31 #include <linux/lustre_mds.h>
32 #include <linux/lustre_idl.h>
33 #include <linux/obd_class.h>
34 #include <linux/obd_lov.h>
35 #include <linux/lustre_lib.h>
36 #include <linux/lustre_fsfilt.h>
38 #include "mds_internal.h"
43 * - magic in mea struct
44 * - error handling is totally missed
47 int mds_lmv_connect(struct obd_device *obd, char * lmv_name)
49 struct mds_obd *mds = &obd->u.mds;
50 struct lustre_handle conn = {0,};
55 if (IS_ERR(mds->mds_lmv_obd))
56 RETURN(PTR_ERR(mds->mds_lmv_obd));
61 mds->mds_lmv_obd = class_name2obd(lmv_name);
62 if (!mds->mds_lmv_obd) {
63 CERROR("MDS cannot locate LMV %s\n",
65 mds->mds_lmv_obd = ERR_PTR(-ENOTCONN);
69 rc = obd_connect(&conn, mds->mds_lmv_obd, &obd->obd_uuid);
71 CERROR("MDS cannot connect to LMV %s (%d)\n",
73 mds->mds_lmv_obd = ERR_PTR(rc);
76 mds->mds_lmv_exp = class_conn2export(&conn);
77 if (mds->mds_lmv_exp == NULL)
78 CERROR("can't get export!\n");
80 rc = obd_register_observer(mds->mds_lmv_obd, obd);
82 CERROR("MDS cannot register as observer of LMV %s (%d)\n",
87 /* retrieve size of EA */
88 rc = obd_get_info(mds->mds_lmv_exp, strlen("mdsize"), "mdsize",
92 if (mdsize > mds->mds_max_mdsize)
93 mds->mds_max_mdsize = mdsize;
95 /* find our number in LMV cluster */
96 rc = obd_get_info(mds->mds_lmv_exp, strlen("mdsnum"), "mdsnum",
100 mds->mds_num = mdsize;
108 /* FIXME: cleanups here! */
109 obd_disconnect(mds->mds_lmv_exp, 0);
110 mds->mds_lmv_exp = NULL;
111 mds->mds_lmv_obd = ERR_PTR(rc);
115 int mds_lmv_postsetup(struct obd_device *obd)
117 struct mds_obd *mds = &obd->u.mds;
119 if (mds->mds_lmv_exp)
120 obd_init_ea_size(mds->mds_lmv_exp, mds->mds_max_mdsize,
121 mds->mds_max_cookiesize);
125 int mds_lmv_disconnect(struct obd_device *obd, int flags)
127 struct mds_obd *mds = &obd->u.mds;
131 if (!IS_ERR(mds->mds_lmv_obd) && mds->mds_lmv_exp != NULL) {
133 obd_register_observer(mds->mds_lmv_obd, NULL);
135 rc = obd_disconnect(mds->mds_lmv_exp, flags);
136 /* if obd_disconnect fails (probably because the
137 * export was disconnected by class_disconnect_exports)
138 * then we just need to drop our ref. */
140 class_export_put(mds->mds_lmv_exp);
141 mds->mds_lmv_exp = NULL;
142 mds->mds_lmv_obd = NULL;
149 int mds_get_lmv_attr(struct obd_device *obd, struct inode *inode,
150 struct mea **mea, int *mea_size)
152 struct mds_obd *mds = &obd->u.mds;
156 if (!mds->mds_lmv_obd)
159 /* first calculate mea size */
160 *mea_size = obd_alloc_diskmd(mds->mds_lmv_exp,
161 (struct lov_mds_md **) mea);
162 /* FIXME: error handling here */
163 LASSERT(*mea != NULL);
166 rc = fsfilt_get_md(obd, inode, *mea, *mea_size);
168 /* FIXME: error handling here */
170 OBD_FREE(*mea, *mea_size);
187 #define DIR_ROUND (DIR_PAD - 1)
188 #define DIR_REC_LEN(name_len) (((name_len) + 12 + DIR_ROUND) & ~DIR_ROUND)
190 /* this struct holds dir entries for particular MDS to be flushed */
192 struct list_head list;
197 struct brw_page brwc;
200 struct dirsplit_control {
201 struct obd_device *obd;
203 struct dentry *dentry;
205 struct dir_cache *cache;
208 static int dc_new_page_to_cache(struct dir_cache * dirc)
212 if (!list_empty(&dirc->list) && dirc->free > sizeof(__u16)) {
213 /* current page became full, mark the end */
214 struct dir_entry *de = dirc->cur;
218 page = alloc_page(GFP_KERNEL);
221 list_add_tail(&page->list, &dirc->list);
222 dirc->cur = page_address(page);
223 dirc->free = PAGE_SIZE;
227 static int retrieve_generation_numbers(struct dirsplit_control *dc, void *buf)
229 struct dir_entry *de;
230 struct dentry *dentry;
233 end = buf + PAGE_SIZE;
234 de = (struct dir_entry *) buf;
235 while ((char *) de < end && de->namelen) {
236 LASSERT(de->namelen <= 255);
237 /* lookup an inode */
238 dentry = ll_lookup_one_len(de->name, dc->dentry, de->namelen);
239 if (IS_ERR(dentry)) {
240 CERROR("can't lookup '%*s'/%u in %lu: %d\n",
241 (int) de->namelen, de->name,
242 (unsigned) de->namelen,
243 (unsigned long) dc->dentry->d_inode->i_ino,
244 (int) PTR_ERR(dentry));
246 LASSERT(!IS_ERR(dentry));
247 LASSERT(dentry->d_inode != NULL);
248 de->generation = dentry->d_inode->i_generation;
250 de = (struct dir_entry *)
251 ((char *) de + DIR_REC_LEN(de->namelen));
256 static int flush_buffer_onto_mds(struct dirsplit_control *dc, int mdsnum)
258 struct mds_obd *mds = &dc->obd->u.mds;
259 struct dir_cache *ca;
260 struct list_head *cur, *tmp;
262 ca = dc->cache + mdsnum;
264 if (ca->free > sizeof(__u16)) {
265 /* current page became full, mark the end */
266 struct dir_entry *de = ca->cur;
270 list_for_each_safe(cur, tmp, &ca->list) {
273 page = list_entry(cur, struct page, list);
274 LASSERT(page != NULL);
276 retrieve_generation_numbers(dc, page_address(page));
280 ca->brwc.count = PAGE_SIZE;
282 ca->oa.o_mds = mdsnum;
283 obd_brw(OBD_BRW_WRITE, mds->mds_lmv_exp, &ca->oa,
284 (struct lov_stripe_md *) dc->mea,
287 list_del(&page->list);
293 static int filldir(void * __buf, const char * name, int namlen, loff_t offset,
294 ino_t ino, unsigned int d_type)
296 struct dirsplit_control *dc = __buf;
297 struct mds_obd *mds = &dc->obd->u.mds;
298 struct dir_cache *ca;
299 struct dir_entry *de;
304 if (name[0] == '.' && (namlen == 1 ||
305 (namlen == 2 && name[1] == '.'))) {
306 /* skip special entries */
311 newmds = mea_name2idx(dc->mea, (char *) name, namlen);
313 if (newmds == mds->mds_num) {
314 /* this entry remains on the current MDS, skip moving */
318 OBD_ALLOC(n, namlen + 1);
319 memcpy(n, name, namlen);
320 n[namlen] = (char) 0;
322 OBD_FREE(n, namlen + 1);
324 /* check for space in buffer for new entry */
325 ca = dc->cache + newmds;
326 if (DIR_REC_LEN(namlen) > ca->free) {
327 int err = dc_new_page_to_cache(ca);
331 /* insert found entry into buffer to be flushed later */
332 /* NOTE: we'll fill generations number later, because we
333 * it's stored in inode, thus we need to lookup an entry,
334 * but directory is locked for readdir(), so we delay this */
338 de->namelen = namlen;
339 memcpy(de->name, name, namlen);
340 ca->cur += DIR_REC_LEN(namlen);
341 ca->free -= DIR_REC_LEN(namlen);
347 int scan_and_distribute(struct obd_device *obd, struct dentry *dentry,
350 struct inode *dir = dentry->d_inode;
351 struct dirsplit_control dc;
356 nlen = strlen("__iopen__/") + 10 + 1;
357 OBD_ALLOC(file_name, nlen);
360 i = sprintf(file_name, "__iopen__/0x%lx", dentry->d_inode->i_ino);
362 file = filp_open(file_name, O_RDONLY, 0);
364 CERROR("can't open directory %s: %d\n",
365 file_name, (int) PTR_ERR(file));
366 OBD_FREE(file_name, nlen);
367 RETURN(PTR_ERR(file));
370 memset(&dc, 0, sizeof(dc));
375 OBD_ALLOC(dc.cache, sizeof(struct dir_cache) * mea->mea_count);
376 LASSERT(dc.cache != NULL);
377 for (i = 0; i < mea->mea_count; i++) {
378 INIT_LIST_HEAD(&dc.cache[i].list);
379 dc.cache[i].free = 0;
380 dc.cache[i].cached = 0;
383 err = vfs_readdir(file, filldir, &dc);
387 for (i = 0; i < mea->mea_count; i++) {
388 if (dc.cache[i].cached)
389 flush_buffer_onto_mds(&dc, i);
392 OBD_FREE(dc.cache, sizeof(struct dir_cache) * mea->mea_count);
393 OBD_FREE(file_name, nlen);
398 #define MAX_DIR_SIZE (64 * 1024)
401 * must not be called on already splitted directories
403 int mds_try_to_split_dir(struct obd_device *obd,
404 struct dentry *dentry, struct mea **mea, int nstripes)
406 struct inode *dir = dentry->d_inode;
407 struct mds_obd *mds = &obd->u.mds;
408 struct mea *tmea = NULL;
409 struct obdo *oa = NULL;
410 int rc, mea_size = 0;
415 if (!mds->mds_lmv_obd)
418 /* don't split root directory */
419 if (dentry->d_inode->i_ino == mds->mds_rootfid.id)
422 /* we want to split only large dirs. this may be already
423 * splitted dir or a slave dir created during splitting */
424 if (dir->i_size < MAX_DIR_SIZE)
427 /* check is directory marked non-splittable */
431 CDEBUG(D_OTHER, "%s: split directory %lu/%lu\n",
432 obd->obd_name, dir->i_ino, (unsigned long) dir->i_generation);
436 mea_size = obd_size_diskmd(mds->mds_lmv_exp, NULL);
438 /* FIXME: Actually we may only want to allocate enough space for
439 * necessary amount of stripes, but on the other hand with this
440 * approach of allocating maximal possible amount of MDS slots,
441 * it would be easier to split the dir over more MDSes */
442 rc = obd_alloc_diskmd(mds->mds_lmv_exp, (void *) mea);
445 (*mea)->mea_count = nstripes;
447 #warning "we have to take EX lock on a dir for splitting"
449 /* 1) create directory objects on slave MDS'es */
450 /* FIXME: should this be OBD method? */
452 /* FIXME: error handling here */
454 oa->o_id = dir->i_ino;
455 oa->o_generation = dir->i_generation;
456 obdo_from_inode(oa, dir, OBD_MD_FLTYPE | OBD_MD_FLATIME |
457 OBD_MD_FLMTIME | OBD_MD_FLCTIME |
458 OBD_MD_FLUID | OBD_MD_FLGID);
459 oa->o_gr = FILTER_GROUP_FIRST_MDS + mds->mds_num;
460 oa->o_valid |= OBD_MD_FLID | OBD_MD_FLFLAGS | OBD_MD_FLGROUP;
461 oa->o_mode = dir->i_mode;
462 CDEBUG(D_OTHER, "%s: create subdirs with mode %o, uid %u, gid %u\n",
463 obd->obd_name, dir->i_mode, dir->i_uid, dir->i_gid);
465 rc = obd_create(mds->mds_lmv_exp, oa,
466 (struct lov_stripe_md **) mea, NULL);
467 /* FIXME: error handling here */
469 CDEBUG(D_OTHER, "%d dirobjects created\n",
470 (int) (*mea)->mea_count);
472 /* 2) update dir attribute */
474 handle = fsfilt_start(obd, dir, FSFILT_OP_SETATTR, NULL);
475 LASSERT(!IS_ERR(handle));
476 rc = fsfilt_set_md(obd, dir, handle, *mea, mea_size);
478 fsfilt_commit(obd, dir, handle, 0);
483 /* 3) read through the dir and distribute it over objects */
484 scan_and_distribute(obd, dentry, *mea);
487 obd_free_diskmd(mds->mds_lmv_exp,
488 (struct lov_mds_md **) mea);
492 static int filter_start_page_write(struct inode *inode,
493 struct niobuf_local *lnb)
495 struct page *page = alloc_pages(GFP_HIGHUSER, 0);
497 CERROR("no memory for a temp page\n");
498 RETURN(lnb->rc = -ENOMEM);
500 POISON_PAGE(page, 0xf1);
501 page->index = lnb->offset >> PAGE_SHIFT;
507 struct dentry *filter_fid2dentry(struct obd_device *obd,
508 struct dentry *dir_dentry,
509 obd_gr group, obd_id id);
510 void f_dput(struct dentry *dentry);
512 int mds_preprw(int cmd, struct obd_export *exp, struct obdo *oa,
513 int objcount, struct obd_ioobj *obj,
514 int niocount, struct niobuf_remote *nb,
515 struct niobuf_local *res,
516 struct obd_trans_info *oti)
518 struct mds_obd *mds = &exp->exp_obd->u.mds;
519 struct niobuf_remote *rnb;
520 struct niobuf_local *lnb = NULL;
521 int rc = 0, i, tot_bytes = 0;
522 unsigned long now = jiffies;
523 struct dentry *dentry;
526 LASSERT(objcount == 1);
527 LASSERT(obj->ioo_bufcnt > 0);
529 memset(res, 0, niocount * sizeof(*res));
531 fid.id = obj->ioo_id;
532 fid.generation = obj->ioo_gr;
533 dentry = mds_fid2dentry(mds, &fid, NULL);
534 LASSERT(!IS_ERR(dentry));
536 if (dentry->d_inode == NULL) {
537 CERROR("trying to BRW to non-existent file "LPU64"\n",
540 GOTO(cleanup, rc = -ENOENT);
543 if (time_after(jiffies, now + 15 * HZ))
544 CERROR("slow preprw_write setup %lus\n", (jiffies - now) / HZ);
546 CDEBUG(D_INFO, "preprw_write setup: %lu jiffies\n",
549 for (i = 0, rnb = nb, lnb = res; i < obj->ioo_bufcnt;
551 lnb->dentry = dentry;
552 lnb->offset = rnb->offset;
554 lnb->flags = rnb->flags;
556 rc = filter_start_page_write(dentry->d_inode, lnb);
558 CDEBUG(rc == -ENOSPC ? D_INODE : D_ERROR, "page err %u@"
559 LPU64" %u/%u %p: rc %d\n", lnb->len, lnb->offset,
560 i, obj->ioo_bufcnt, dentry, rc);
562 __free_pages(lnb->page, 0);
566 tot_bytes += lnb->len;
569 if (time_after(jiffies, now + 15 * HZ))
570 CERROR("slow start_page_write %lus\n", (jiffies - now) / HZ);
572 CDEBUG(D_INFO, "start_page_write: %lu jiffies\n",
580 int mds_commitrw(int cmd, struct obd_export *exp, struct obdo *oa,
581 int objcount, struct obd_ioobj *obj, int niocount,
582 struct niobuf_local *res, struct obd_trans_info *oti,
585 struct obd_device *obd = exp->exp_obd;
586 struct niobuf_local *lnb;
587 struct inode *inode = NULL;
588 int rc = 0, i, cleanup_phase = 0, err, entries = 0;
591 LASSERT(objcount == 1);
592 LASSERT(current->journal_info == NULL);
595 inode = res->dentry->d_inode;
597 for (i = 0, lnb = res; i < obj->ioo_bufcnt; i++, lnb++) {
599 struct dir_entry *de;
601 buf = kmap(lnb->page);
602 LASSERT(buf != NULL);
603 end = buf + lnb->len;
604 de = (struct dir_entry *) buf;
605 while ((char *) de < end && de->namelen) {
606 err = fsfilt_add_dir_entry(obd, res->dentry, de->name,
607 de->namelen, de->ino,
608 de->generation, de->mds);
609 /* FIXME: remove entries from the original dir */
610 #warning "removing entries from the original dir"
612 de = (struct dir_entry *)
613 ((char *) de + DIR_REC_LEN(de->namelen));
619 for (i = 0, lnb = res; i < obj->ioo_bufcnt; i++, lnb++)
620 __free_page(lnb->page);
626 int mds_choose_mdsnum(struct obd_device *obd, const char *name, int len)
628 struct mds_obd *mds = &obd->u.mds;
629 struct lmv_obd *lmv = &mds->mds_lmv_exp->exp_obd->u.lmv;
632 i = raw_name2idx(lmv->count, name, len);