X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Fmds%2Fmds_lov.c;h=70ab89c8ccc71018950d3d8da623427a3fa96172;hb=0517160dd68ac026513ad1b8e3e6f7abd4acfdef;hp=0f6bf52091a1e470c581011feb072e4549dee1ad;hpb=efd018086983067ac27a44962f01fc82eb7e6b96;p=fs%2Flustre-release.git diff --git a/lustre/mds/mds_lov.c b/lustre/mds/mds_lov.c index 0f6bf52..70ab89c 100644 --- a/lustre/mds/mds_lov.c +++ b/lustre/mds/mds_lov.c @@ -1,953 +1,1034 @@ /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- * vim:expandtab:shiftwidth=8:tabstop=8: * - * linux/mds/mds_lov.c - * Lustre Metadata Server (mds) handling of striped file data + * GPL HEADER START * - * Copyright (C) 2001-2003 Cluster File Systems, Inc. - * Author: Peter Braam + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * - * This file is part of Lustre, http://www.lustre.org. + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. * - * Lustre is free software; you can redistribute it and/or - * modify it under the terms of version 2 of the GNU General Public - * License as published by the Free Software Foundation. + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). * - * Lustre is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * You should have received a copy of the GNU General Public License + * version 2 along with this program; If not, see + * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf * - * You should have received a copy of the GNU General Public License - * along with Lustre; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, + * CA 95054 USA or visit www.sun.com if you need additional information or + * have any questions. + * + * GPL HEADER END + */ +/* + * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved. + * Use is subject to license terms. + */ +/* + * This file is part of Lustre, http://www.lustre.org/ + * Lustre is a trademark of Sun Microsystems, Inc. + * + * lustre/mds/mds_lov.c + * + * Lustre Metadata Server (mds) handling of striped file data + * + * Author: Peter Braam */ -#ifndef EXPORT_SYMTAB -# define EXPORT_SYMTAB -#endif #define DEBUG_SUBSYSTEM S_MDS #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include "mds_internal.h" -void le_lov_desc_to_cpu (struct lov_desc *ld) +static void mds_lov_dump_objids(const char *label, struct obd_device *obd) { - ld->ld_tgt_count = le32_to_cpu (ld->ld_tgt_count); - ld->ld_default_stripe_count = le32_to_cpu (ld->ld_default_stripe_count); - ld->ld_default_stripe_size = le32_to_cpu (ld->ld_default_stripe_size); - ld->ld_pattern = le32_to_cpu (ld->ld_pattern); -} + struct mds_obd *mds = &obd->u.mds; + unsigned int i=0, j; -void cpu_to_le_lov_desc (struct lov_desc *ld) -{ - ld->ld_tgt_count = cpu_to_le32 (ld->ld_tgt_count); - ld->ld_default_stripe_count = cpu_to_le32 (ld->ld_default_stripe_count); - ld->ld_default_stripe_size = cpu_to_le32 (ld->ld_default_stripe_size); - ld->ld_pattern = cpu_to_le32 (ld->ld_pattern); -} + if ((libcfs_debug & D_INFO) == 0) + return; -void mds_dt_update_objids(struct obd_device *obd, obd_id *ids) -{ - struct mds_obd *mds = &obd->u.mds; - int i; - ENTRY; + CDEBUG(D_INFO, "dump from %s\n", label); + if (mds->mds_lov_page_dirty == NULL) { + CERROR("NULL bitmap!\n"); + GOTO(skip_bitmap, i); + } + + for(i = 0; i < mds->mds_lov_page_dirty->size / BITS_PER_LONG + 1; i++) + CDEBUG(D_INFO, "%u - %lx\n", i, + mds->mds_lov_page_dirty->data[i]); +skip_bitmap: + if (mds->mds_lov_page_array == NULL) { + CERROR("not init page array!\n"); + GOTO(skip_array, i); + + } + for(i = 0;i < MDS_LOV_OBJID_PAGES_COUNT; i++) { + obd_id *data = mds->mds_lov_page_array[i]; - spin_lock(&mds->mds_dt_lock); - for (i = 0; i < mds->mds_dt_desc.ld_tgt_count; i++) - if (ids[i] > (mds->mds_dt_objids)[i]) - (mds->mds_dt_objids)[i] = ids[i]; - spin_unlock(&mds->mds_dt_lock); + if (data == NULL) + continue; + + for(j=0; j < OBJID_PER_PAGE(); j++) { + if (data[j] == 0) + continue; + CDEBUG(D_INFO,"objid page %u idx %u - "LPU64" \n", + i, j, data[j]); + } + } +skip_array: EXIT; } -static int mds_dt_read_objids(struct obd_device *obd) +int mds_lov_init_objids(struct obd_device *obd) { struct mds_obd *mds = &obd->u.mds; - obd_id *ids; - loff_t off = 0; - int i, rc, size = mds->mds_dt_desc.ld_tgt_count * sizeof(*ids); + int size = MDS_LOV_OBJID_PAGES_COUNT*sizeof(void *); + struct file *file; + int rc; ENTRY; - if (mds->mds_dt_objids != NULL) - RETURN(0); + CLASSERT(((MDS_LOV_ALLOC_SIZE % sizeof(obd_id)) == 0)); - OBD_ALLOC(ids, size); - if (ids == NULL) + mds->mds_lov_page_dirty = + CFS_ALLOCATE_BITMAP(MDS_LOV_OBJID_PAGES_COUNT); + if (mds->mds_lov_page_dirty == NULL) RETURN(-ENOMEM); - mds->mds_dt_objids = ids; - if (mds->mds_dt_objid_filp->f_dentry->d_inode->i_size == 0) - RETURN(0); - rc = fsfilt_read_record(obd, mds->mds_dt_objid_filp, ids, size, &off); - if (rc < 0) { - CERROR("Error reading objids %d\n", rc); - } else { - mds->mds_dt_objids_valid = 1; - rc = 0; + + OBD_ALLOC(mds->mds_lov_page_array, size); + if (mds->mds_lov_page_array == NULL) + GOTO(err_free_bitmap, rc = -ENOMEM); + + /* open and test the lov objd file */ + file = filp_open(LOV_OBJID, O_RDWR | O_CREAT, 0644); + if (IS_ERR(file)) { + rc = PTR_ERR(file); + CERROR("cannot open/create %s file: rc = %d\n", LOV_OBJID, rc); + GOTO(err_free, rc = PTR_ERR(file)); } + if (!S_ISREG(file->f_dentry->d_inode->i_mode)) { + CERROR("%s is not a regular file!: mode = %o\n", LOV_OBJID, + file->f_dentry->d_inode->i_mode); + GOTO(err_open, rc = -ENOENT); + } + mds->mds_lov_objid_filp = file; - for (i = 0; i < mds->mds_dt_desc.ld_tgt_count; i++) - CDEBUG(D_INFO, "read last object "LPU64" for idx %d\n", - mds->mds_dt_objids[i], i); + RETURN (0); +err_open: + if (filp_close((struct file *)file, 0)) + CERROR("can't close %s after error\n", LOV_OBJID); +err_free: + OBD_FREE(mds->mds_lov_page_array, size); +err_free_bitmap: + CFS_FREE_BITMAP(mds->mds_lov_page_dirty); RETURN(rc); } -int mds_dt_write_objids(struct obd_device *obd) +void mds_lov_destroy_objids(struct obd_device *obd) { struct mds_obd *mds = &obd->u.mds; - loff_t off = 0; - int i, rc, size = mds->mds_dt_desc.ld_tgt_count * sizeof(obd_id); + int i, rc; ENTRY; - for (i = 0; i < mds->mds_dt_desc.ld_tgt_count; i++) - CDEBUG(D_INFO, "writing last object "LPU64" for idx %d\n", - mds->mds_dt_objids[i], i); + if (mds->mds_lov_page_array != NULL) { + for(i=0;imds_lov_page_array[i]; + if (data != NULL) + OBD_FREE(data, MDS_LOV_ALLOC_SIZE); + } + OBD_FREE(mds->mds_lov_page_array, + MDS_LOV_OBJID_PAGES_COUNT*sizeof(void *)); + } - rc = fsfilt_write_record(obd, mds->mds_dt_objid_filp, - mds->mds_dt_objids, size, &off, 0); - RETURN(rc); + if (mds->mds_lov_objid_filp) { + rc = filp_close((struct file *)mds->mds_lov_objid_filp, NULL); + mds->mds_lov_objid_filp = NULL; + if (rc) + CERROR("%s file won't close, rc=%d\n", LOV_OBJID, rc); + } + + CFS_FREE_BITMAP(mds->mds_lov_page_dirty); + EXIT; } -int mds_dt_clearorphans(struct mds_obd *mds, struct obd_uuid *ost_uuid) +/** + * currently exist two ways for know about ost count and max ost index. + * first - after ost is connected to mds and sync process finished + * second - get from lmm in recovery process, in case when mds not have configs, + * and ost isn't registered in mgs. + * + * \param mds pointer to mds structure + * \param index maxium ost index + * + * \retval -ENOMEM is not hame memory for new page + * \retval 0 is update passed + */ +static int mds_lov_update_max_ost(struct mds_obd *mds, obd_id index) { - int rc; - struct obdo *oa = NULL; - struct obd_trans_info oti = {0}; - struct lov_stripe_md *empty_ea = NULL; - ENTRY; - - LASSERT(mds->mds_dt_objids != NULL); + __u32 page = index / OBJID_PER_PAGE(); + __u32 off = index % OBJID_PER_PAGE(); + obd_id *data = mds->mds_lov_page_array[page]; - /* - * this create will in fact either create or destroy: If the OST is - * missing objects below this ID, they will be created. If it finds - * objects above this ID, they will be removed. - */ - oa = obdo_alloc(); - if (oa == NULL) - RETURN(-ENOMEM); - - memset(oa, 0, sizeof(*oa)); - oa->o_gr = FILTER_GROUP_FIRST_MDS + mds->mds_num; - oa->o_valid = OBD_MD_FLFLAGS | OBD_MD_FLGROUP; - oa->o_flags = OBD_FL_DELORPHAN; - - if (ost_uuid != NULL) { - memcpy(&oa->o_inline, ost_uuid, sizeof(*ost_uuid)); - oa->o_valid |= OBD_MD_FLINLINE; - } - rc = obd_create(mds->mds_dt_exp, oa, NULL, 0, &empty_ea, &oti); - obdo_free(oa); - RETURN(rc); -} + if (data == NULL) { + OBD_ALLOC(data, MDS_LOV_ALLOC_SIZE); + if (data == NULL) + RETURN(-ENOMEM); -/* update the LOV-OSC knowledge of the last used object id's */ -int mds_dt_set_nextid(struct obd_device *obd) -{ - struct mds_obd *mds = &obd->u.mds; - int rc; - ENTRY; + mds->mds_lov_page_array[page] = data; + } - LASSERT(!obd->obd_recovering); + if (index > mds->mds_lov_objid_max_index) { + mds->mds_lov_objid_lastpage = page; + mds->mds_lov_objid_lastidx = off; + mds->mds_lov_objid_max_index = index; + } - LASSERT(mds->mds_dt_objids != NULL); + /* workaround - New target not in objids file; increase mdsize */ + /* ld_tgt_count is used as the max index everywhere, despite its name. */ + if (data[off] == 0) { + __u32 stripes; - rc = obd_set_info(mds->mds_dt_exp, strlen("next_id"), "next_id", - mds->mds_dt_desc.ld_tgt_count, mds->mds_dt_objids); - RETURN(rc); -} + data[off] = 1; + mds->mds_lov_objid_count++; + stripes = min_t(__u32, LOV_MAX_STRIPE_COUNT, + mds->mds_lov_objid_count); -/* tell the LOV-OSC by how much to pre-create */ -int mds_dt_set_growth(struct mds_obd *mds, int count) -{ - int rc; - ENTRY; + mds->mds_max_mdsize = lov_mds_md_size(stripes, LOV_MAGIC_V3); + mds->mds_max_cookiesize = stripes * sizeof(struct llog_cookie); - rc = obd_set_info(mds->mds_dt_exp, strlen("growth_count"), - "growth_count", sizeof(count), &count); + CDEBUG(D_CONFIG, "updated max_mdsize/max_cookiesize for %d" + " stripes: %d/%d\n", stripes, mds->mds_max_mdsize, + mds->mds_max_cookiesize); + } - RETURN(rc); + EXIT; + return 0; } -static int mds_dt_update_desc(struct obd_device *obd, struct obd_export *lov) +static int mds_lov_objinit(struct mds_obd *mds, __u32 index) { - struct mds_obd *mds = &obd->u.mds; - int valsize = sizeof(mds->mds_dt_desc), rc, i; - int old_count = mds->mds_dt_desc.ld_tgt_count; - ENTRY; - - rc = obd_get_info(lov, strlen("lovdesc") + 1, "lovdesc", &valsize, - &mds->mds_dt_desc); - if (rc) - RETURN(rc); - - /* The size of the LOV target table may have increased. */ - if (old_count >= mds->mds_dt_desc.ld_tgt_count) { - obd_id *ids; - int size; + __u32 page = index / OBJID_PER_PAGE(); + __u32 off = index % OBJID_PER_PAGE(); + obd_id *data = mds->mds_lov_page_array[page]; - size = mds->mds_dt_desc.ld_tgt_count * sizeof(*ids); - OBD_ALLOC(ids, size); - if (ids == NULL) - RETURN(-ENOMEM); + return (data[off] > 0); +} - memset(ids, 0, size); +int mds_lov_prepare_objids(struct obd_device *obd, struct lov_mds_md *lmm) +{ + struct lov_ost_data_v1 *data; + __u32 count; + int rc = 0; + __u32 j; + + /* if we create file without objects - lmm is NULL */ + if (lmm == NULL) + return 0; + + switch (le32_to_cpu(lmm->lmm_magic)) { + case LOV_MAGIC_V1: + count = le32_to_cpu(((struct lov_mds_md_v1*)lmm)->lmm_stripe_count); + data = &(((struct lov_mds_md_v1*)lmm)->lmm_objects[0]); + break; + case LOV_MAGIC_V3: + count = le32_to_cpu(((struct lov_mds_md_v3*)lmm)->lmm_stripe_count); + data = &(((struct lov_mds_md_v3*)lmm)->lmm_objects[0]); + break; + default: + CERROR("Unknow lmm type %X!\n", le32_to_cpu(lmm->lmm_magic)); + RETURN(-EINVAL); + } - if (mds->mds_dt_objids != NULL) { - int oldsize = old_count * sizeof(*ids); - memcpy(ids, mds->mds_dt_objids, oldsize); - OBD_FREE(mds->mds_dt_objids, oldsize); + cfs_mutex_down(&obd->obd_dev_sem); + for (j = 0; j < count; j++) { + __u32 i = le32_to_cpu(data[j].l_ost_idx); + if (mds_lov_update_max_ost(&obd->u.mds, i)) { + rc = -ENOMEM; + break; } - mds->mds_dt_objids = ids; } + cfs_mutex_up(&obd->obd_dev_sem); - i = lov_mds_md_size(mds->mds_dt_desc.ld_tgt_count); - if (i > mds->mds_max_mdsize) - mds->mds_max_mdsize = i; - mds->mds_max_cookiesize = mds->mds_dt_desc.ld_tgt_count * - sizeof(struct llog_cookie); - mds->mds_has_dt_desc = 1; - RETURN(0); + RETURN(rc); } +EXPORT_SYMBOL(mds_lov_prepare_objids); -int mds_dt_connect(struct obd_device *obd, char * lov_name) +/* + * write llog orphan record about lost ost object, + * Special lsm is allocated with single stripe, caller should deallocated it + * after use + */ +static int mds_log_lost_precreated(struct obd_device *obd, + struct lov_stripe_md **lsmp, int *stripes, + obd_id id, obd_count count, int idx) { - struct mds_obd *mds = &obd->u.mds; - struct lustre_handle conn = {0,}; - int rc, i; + struct lov_stripe_md *lsm = *lsmp; + int rc; ENTRY; - if (IS_ERR(mds->mds_dt_obd)) - RETURN(PTR_ERR(mds->mds_dt_obd)); - - if (mds->mds_dt_obd) - RETURN(0); - - spin_lock_init(&mds->mds_dt_lock); - mds->mds_dt_obd = class_name2obd(lov_name); - if (!mds->mds_dt_obd) { - CERROR("MDS cannot locate LOV %s\n", lov_name); - mds->mds_dt_obd = ERR_PTR(-ENOTCONN); - RETURN(-ENOTCONN); - } - - if (mds->mds_ost_sec) { - rc = obd_set_info(mds->mds_dt_obd->obd_self_export, - strlen("sec"), "sec", - strlen(mds->mds_ost_sec), mds->mds_ost_sec); - if (rc) { - mds->mds_dt_obd = ERR_PTR(rc); + if (*lsmp == NULL) { + rc = obd_alloc_memmd(obd->u.mds.mds_lov_exp, &lsm); + if (rc < 0) RETURN(rc); - } + /* need only one stripe, save old value */ + *stripes = lsm->lsm_stripe_count; + lsm->lsm_stripe_count = 1; + *lsmp = lsm; } - CDEBUG(D_HA, "obd: %s osc: %s lov_name: %s\n", - obd->obd_name, mds->mds_dt_obd->obd_name, lov_name); - - rc = obd_connect(&conn, mds->mds_dt_obd, &obd->obd_uuid, NULL, - mds->mds_num + FILTER_GROUP_FIRST_MDS); - if (rc) { - CERROR("MDS cannot connect to LOV %s (%d)\n", lov_name, rc); - mds->mds_dt_obd = ERR_PTR(rc); - RETURN(rc); - } - mds->mds_dt_exp = class_conn2export(&conn); + lsm->lsm_oinfo[0]->loi_id = id; + lsm->lsm_oinfo[0]->loi_seq = mdt_to_obd_objseq(obd->u.mds.mds_id); + lsm->lsm_oinfo[0]->loi_ost_idx = idx; - rc = obd_register_observer(mds->mds_dt_obd, obd); - if (rc) { - CERROR("MDS cannot register as observer of LOV %s (%d)\n", - lov_name, rc); - GOTO(err_discon, rc); - } + rc = mds_log_op_orphan(obd, lsm, count); + RETURN(rc); +} - rc = mds_dt_update_desc(obd, mds->mds_dt_exp); - if (rc) - GOTO(err_reg, rc); +void mds_lov_update_objids(struct obd_device *obd, struct lov_mds_md *lmm) +{ + struct mds_obd *mds = &obd->u.mds; + int j; + struct lov_ost_data_v1 *obj; + struct lov_stripe_md *lsm = NULL; + int stripes = 0; + int count; + ENTRY; - rc = mds_dt_read_objids(obd); - if (rc) { - CERROR("cannot read %s: rc = %d\n", "lov_objids", rc); - GOTO(err_reg, rc); + /* if we create file without objects - lmm is NULL */ + if (lmm == NULL) + return; + + switch (le32_to_cpu(lmm->lmm_magic)) { + case LOV_MAGIC_V1: + count = le32_to_cpu(((struct lov_mds_md_v1*)lmm)->lmm_stripe_count); + obj = ((struct lov_mds_md_v1*)lmm)->lmm_objects; + break; + case LOV_MAGIC_V3: + count = le32_to_cpu(((struct lov_mds_md_v3*)lmm)->lmm_stripe_count); + obj = ((struct lov_mds_md_v3*)lmm)->lmm_objects; + break; + default: + CERROR("Unknow lmm type %X !\n", + le32_to_cpu(lmm->lmm_magic)); + return; } - rc = obd_llog_cat_initialize(obd, &obd->obd_llogs, - mds->mds_dt_desc.ld_tgt_count, CATLIST); - if (rc) { - CERROR("failed to initialize catalog %d\n", rc); - GOTO(err_reg, rc); - } - - /* If we're mounting this code for the first time on an existing FS, - * we need to populate the objids array from the real OST values */ - if (!mds->mds_dt_objids_valid) { - int size = sizeof(obd_id) * mds->mds_dt_desc.ld_tgt_count; - rc = obd_get_info(mds->mds_dt_exp, strlen("last_id"), - "last_id", &size, mds->mds_dt_objids); - if (!rc) { - for (i = 0; i < mds->mds_dt_desc.ld_tgt_count; i++) - CWARN("got last object "LPU64" from OST %d\n", - mds->mds_dt_objids[i], i); - mds->mds_dt_objids_valid = 1; - rc = mds_dt_write_objids(obd); - if (rc) - CERROR("got last objids from OSTs, but error " - "writing objids file: %d\n", rc); + for (j = 0; j < count; j++) { + __u32 i = le32_to_cpu(obj[j].l_ost_idx); + obd_id id = le64_to_cpu(obj[j].l_object_id); + __u32 page = i / OBJID_PER_PAGE(); + __u32 idx = i % OBJID_PER_PAGE(); + obd_id *data; + + data = mds->mds_lov_page_array[page]; + + CDEBUG(D_INODE,"update last object for ost %u" + " - new "LPU64" old "LPU64"\n", i, id, data[idx]); + if (id > data[idx]) { + int lost = id - data[idx] - 1; + /* we might have lost precreated objects due to VBR */ + if (lost > 0 && obd->obd_recovering) { + CDEBUG(D_HA, "Gap in objids is %u\n", lost); + if (!obd->obd_version_recov) + CERROR("Unexpected gap in objids\n"); + /* lsm is allocated if NULL */ + mds_log_lost_precreated(obd, &lsm, &stripes, + data[idx]+1, lost, i); + } + data[idx] = id; + cfs_bitmap_set(mds->mds_lov_page_dirty, page); } } - - /* I want to see a callback happen when the OBD moves to a - * "For General Use" state, and that's when we'll call - * set_nextid(). The class driver can help us here, because - * it can use the obd_recovering flag to determine when the - * the OBD is full available. */ - if (!obd->obd_recovering) { - CDEBUG(D_OTHER, "call mds_postrecov_common()\n"); - rc = mds_postrecov_common(obd); - if (rc > 0) - rc = 0; + if (lsm) { + /* restore stripes number */ + lsm->lsm_stripe_count = stripes; + obd_free_memmd(mds->mds_lov_exp, &lsm); } - RETURN(rc); - -err_reg: - obd_register_observer(mds->mds_dt_obd, NULL); -err_discon: - obd_disconnect(mds->mds_dt_exp, 0); - mds->mds_dt_exp = NULL; - mds->mds_dt_obd = ERR_PTR(rc); - return rc; + EXIT; + return; } +EXPORT_SYMBOL(mds_lov_update_objids); -int mds_dt_disconnect(struct obd_device *obd, int flags) +static int mds_lov_update_from_read(struct mds_obd *mds, obd_id *data, + __u32 count) { - struct mds_obd *mds = &obd->u.mds; - int rc = 0; - ENTRY; - - if (!IS_ERR(mds->mds_dt_obd) && mds->mds_dt_exp != NULL) { - /* cleanup all llogging subsystems */ - rc = obd_llog_finish(obd, &obd->obd_llogs, - mds->mds_dt_desc.ld_tgt_count); - if (rc) - CERROR("failed to cleanup llogging subsystems\n"); + __u32 i; + __u32 stripes; - obd_register_observer(mds->mds_dt_obd, NULL); + for (i = 0; i < count; i++) { + if (data[i] == 0) + continue; - rc = obd_disconnect(mds->mds_dt_exp, flags); - /* if obd_disconnect fails (probably because the - * export was disconnected by class_disconnect_exports) - * then we just need to drop our ref. */ - if (rc != 0) - class_export_put(mds->mds_dt_exp); - mds->mds_dt_exp = NULL; - mds->mds_dt_obd = NULL; + mds->mds_lov_objid_count++; } - RETURN(rc); + stripes = min_t(__u32, LOV_MAX_STRIPE_COUNT, + mds->mds_lov_objid_count); + + mds->mds_max_mdsize = lov_mds_md_size(stripes, LOV_MAGIC_V3); + mds->mds_max_cookiesize = stripes * sizeof(struct llog_cookie); + + CDEBUG(D_CONFIG, "updated max_mdsize/max_cookiesize for %d stripes: " + "%d/%d\n", stripes, mds->mds_max_mdsize, + mds->mds_max_cookiesize); + + EXIT; + return 0; } -int mds_iocontrol(unsigned int cmd, struct obd_export *exp, int len, - void *karg, void *uarg) +static int mds_lov_read_objids(struct obd_device *obd) { - static struct obd_uuid cfg_uuid = { .uuid = "config_uuid" }; - struct obd_device *obd = exp->exp_obd; struct mds_obd *mds = &obd->u.mds; - struct obd_ioctl_data *data = karg; - struct lvfs_run_ctxt saved; - int rc = 0; + loff_t off = 0; + int i, rc = 0, count = 0, page = 0; + unsigned long size; ENTRY; - CDEBUG(D_INFO, "ioctl cmd %x\n", cmd); - switch (cmd) { - case OBD_IOC_RECORD: { - char *name = data->ioc_inlbuf1; - if (mds->mds_cfg_llh) - RETURN(-EBUSY); - - push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - rc = llog_open(llog_get_context(&obd->obd_llogs, - LLOG_CONFIG_ORIG_CTXT), - &mds->mds_cfg_llh, NULL, name, - OBD_LLOG_FL_CREATE); - if (rc == 0) - llog_init_handle(mds->mds_cfg_llh, LLOG_F_IS_PLAIN, - &cfg_uuid); - else - mds->mds_cfg_llh = NULL; - pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - - RETURN(rc); - } - - case OBD_IOC_ENDRECORD: { - if (!mds->mds_cfg_llh) - RETURN(-EBADF); - - push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - rc = llog_close(mds->mds_cfg_llh); - pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + /* Read everything in the file, even if our current lov desc + has fewer targets. Old targets not in the lov descriptor + during mds setup may still have valid objids. */ + size = i_size_read(mds->mds_lov_objid_filp->f_dentry->d_inode); + if (size == 0) + RETURN(0); - mds->mds_cfg_llh = NULL; - RETURN(rc); - } + page = (size + MDS_LOV_ALLOC_SIZE - 1) / MDS_LOV_ALLOC_SIZE; + CDEBUG(D_INFO, "file size %lu pages %d\n", size, page); + for (i = 0; i < page; i++) { + obd_id *data; + loff_t off_old = off; - case OBD_IOC_CLEAR_LOG: { - char *name = data->ioc_inlbuf1; - if (mds->mds_cfg_llh) - RETURN(-EBUSY); + LASSERT(mds->mds_lov_page_array[i] == NULL); + OBD_ALLOC(mds->mds_lov_page_array[i], MDS_LOV_ALLOC_SIZE); + if (mds->mds_lov_page_array[i] == NULL) + GOTO(out, rc = -ENOMEM); - push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - rc = llog_open(llog_get_context(&obd->obd_llogs, - LLOG_CONFIG_ORIG_CTXT), - &mds->mds_cfg_llh, NULL, name, - OBD_LLOG_FL_CREATE); - if (rc == 0) { - llog_init_handle(mds->mds_cfg_llh, LLOG_F_IS_PLAIN, - NULL); + data = mds->mds_lov_page_array[i]; - rc = llog_destroy(mds->mds_cfg_llh); - llog_free_handle(mds->mds_cfg_llh); + rc = fsfilt_read_record(obd, mds->mds_lov_objid_filp, data, + MDS_LOV_ALLOC_SIZE, &off); + if (rc < 0) { + CERROR("Error reading objids %d\n", rc); + GOTO(out, rc); } - pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + if (off == off_old) /* hole is read */ + off += MDS_LOV_ALLOC_SIZE; - mds->mds_cfg_llh = NULL; - RETURN(rc); + count = (off - off_old) / sizeof(obd_id); + if (mds_lov_update_from_read(mds, data, count)) { + CERROR("Can't update mds data\n"); + GOTO(out, rc = -EIO); + } } + mds->mds_lov_objid_lastpage = page - 1; + mds->mds_lov_objid_lastidx = count - 1; - case OBD_IOC_DORECORD: { - char *cfg_buf; - struct llog_rec_hdr rec; - if (!mds->mds_cfg_llh) - RETURN(-EBADF); + CDEBUG(D_INFO, "Read %u - %u %u objid\n", mds->mds_lov_objid_count, + mds->mds_lov_objid_lastpage, mds->mds_lov_objid_lastidx); +out: + mds_lov_dump_objids("read",obd); - /* XXX - this probably should be a parameter to this ioctl. - * For now, just use llh_max_transno for expediency. */ - rec.lrh_len = llog_data_len(data->ioc_plen1); + RETURN(rc); +} - if (data->ioc_type == LUSTRE_CFG_TYPE) { - rec.lrh_type = OBD_CFG_REC; - } else if (data->ioc_type == PORTALS_CFG_TYPE) { - rec.lrh_type = PTL_CFG_REC; - } else { - CERROR("unknown cfg record type:%d \n", data->ioc_type); - RETURN(-EINVAL); - } +int mds_lov_write_objids(struct obd_device *obd) +{ + struct mds_obd *mds = &obd->u.mds; + int i = 0, rc = 0; + ENTRY; - OBD_ALLOC(cfg_buf, data->ioc_plen1); - if (cfg_buf == NULL) - RETURN(-EINVAL); - rc = copy_from_user(cfg_buf, data->ioc_pbuf1, data->ioc_plen1); - if (rc) { - OBD_FREE(cfg_buf, data->ioc_plen1); - RETURN(rc); - } + if (cfs_bitmap_check_empty(mds->mds_lov_page_dirty)) + RETURN(0); - push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - rc = llog_write_rec(mds->mds_cfg_llh, &rec, NULL, 0, - cfg_buf, -1); - pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + mds_lov_dump_objids("write", obd); - OBD_FREE(cfg_buf, data->ioc_plen1); - RETURN(rc); - } - case OBD_IOC_PARSE: { - struct llog_ctxt *ctxt = - llog_get_context(&obd->obd_llogs, LLOG_CONFIG_ORIG_CTXT); - push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - rc = class_config_process_llog(ctxt, data->ioc_inlbuf1, NULL); - pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - if (rc) - RETURN(rc); + cfs_foreach_bit(mds->mds_lov_page_dirty, i) { + obd_id *data = mds->mds_lov_page_array[i]; + unsigned int size = MDS_LOV_ALLOC_SIZE; + loff_t off = i * size; - RETURN(rc); - } - case OBD_IOC_DUMP_LOG: { - struct llog_ctxt *ctxt = - llog_get_context(&obd->obd_llogs, LLOG_CONFIG_ORIG_CTXT); - push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - rc = class_config_dump_llog(ctxt, data->ioc_inlbuf1, NULL); - pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + LASSERT(data != NULL); - RETURN(rc); - } - case OBD_IOC_SET_READONLY: { - void *handle; - struct inode *inode = obd->u.mds.mds_sb->s_root->d_inode; - BDEVNAME_DECLARE_STORAGE(tmp); - CERROR("setting device %s read-only\n", - ll_bdevname(obd->u.mds.mds_sb, tmp)); + if (!cfs_bitmap_test_and_clear(mds->mds_lov_page_dirty, i)) + continue; - handle = fsfilt_start(obd, inode, FSFILT_OP_MKNOD, NULL); - LASSERT(handle); - rc = fsfilt_commit(obd, obd->u.mds.mds_sb, inode, handle, 1); + /* check for particaly filled last page */ + if (i == mds->mds_lov_objid_lastpage) + size = (mds->mds_lov_objid_lastidx+1) * sizeof(obd_id); - ll_set_rdonly(ll_sbdev(obd->u.mds.mds_sb)); - RETURN(0); + CDEBUG(D_INFO, "write %lld - %u\n", off, size); + rc = fsfilt_write_record(obd, mds->mds_lov_objid_filp, data, + size, &off, 0); + if (rc < 0) { + cfs_bitmap_set(mds->mds_lov_page_dirty, i); + break; + } } + if (rc >= 0) + rc = 0; - case OBD_IOC_CATLOGLIST: { - int count = mds->mds_dt_desc.ld_tgt_count; - rc = llog_catalog_list(obd, count, data); - RETURN(rc); + RETURN(rc); +} +EXPORT_SYMBOL(mds_lov_write_objids); - } - case OBD_IOC_LLOG_CHECK: - case OBD_IOC_LLOG_CANCEL: - case OBD_IOC_LLOG_REMOVE: { - struct llog_ctxt *ctxt = - llog_get_context(&obd->obd_llogs, LLOG_CONFIG_ORIG_CTXT); - int rc2, valsize; - __u32 group; - - obd_llog_finish(obd, &obd->obd_llogs, - mds->mds_dt_desc.ld_tgt_count); - push_ctxt(&saved, ctxt->loc_lvfs_ctxt, NULL); - rc = llog_ioctl(ctxt, cmd, data); - pop_ctxt(&saved, ctxt->loc_lvfs_ctxt, NULL); - obd_llog_cat_initialize(obd, &obd->obd_llogs, - mds->mds_dt_desc.ld_tgt_count, - CATLIST); - group = FILTER_GROUP_FIRST_MDS + mds->mds_num; - valsize = sizeof(group); - rc2 = obd_set_info(mds->mds_dt_exp, strlen("mds_conn"), - "mds_conn", valsize, &group); - if (!rc) - rc = rc2; - RETURN(rc); - } - case OBD_IOC_LLOG_INFO: - case OBD_IOC_LLOG_PRINT: { - struct llog_ctxt *ctxt = - llog_get_context(&obd->obd_llogs, LLOG_CONFIG_ORIG_CTXT); +static int mds_lov_get_objid(struct obd_device * obd, + obd_id idx) +{ + struct mds_obd *mds = &obd->u.mds; + struct obd_export *lov_exp = mds->mds_lov_exp; + unsigned int page; + unsigned int off; + obd_id *data; + __u32 size; + int rc = 0; + ENTRY; - push_ctxt(&saved, ctxt->loc_lvfs_ctxt, NULL); - rc = llog_ioctl(ctxt, cmd, data); - pop_ctxt(&saved, ctxt->loc_lvfs_ctxt, NULL); + page = idx / OBJID_PER_PAGE(); + off = idx % OBJID_PER_PAGE(); + data = mds->mds_lov_page_array[page]; - RETURN(rc); - } + if (data[off] < 2) { + /* We never read this lastid; ask the osc */ + struct obd_id_info lastid; - case OBD_IOC_ABORT_RECOVERY: - target_stop_recovery_thread(obd); - RETURN(0); - case OBD_IOC_ROOT_SQUASH: { - __u32 *p = (__u32 *) data->ioc_inlbuf1; + size = sizeof(lastid); + lastid.idx = idx; + lastid.data = &data[off]; + rc = obd_get_info(lov_exp, sizeof(KEY_LAST_ID), KEY_LAST_ID, + &size, &lastid, NULL); + if (rc) + GOTO(out, rc); - if (data->ioc_inllen1 != - (sizeof(__u32) * 4 + sizeof(ptl_nid_t))) - RETURN(-EINVAL); + /* workaround for clean filter */ + if (data[off] == 0) + data[off] = 1; - if (*p == 0) { /* get */ - p += 2; - *p++ = mds->mds_squash_uid; - *p++ = mds->mds_squash_gid; - *((ptl_nid_t*) p) = mds->mds_nosquash_nid; - } else { /* set */ - p += 2; - mds->mds_squash_uid = *p++; - mds->mds_squash_gid = *p++; - mds->mds_nosquash_nid = *((ptl_nid_t*) p); - CWARN("MDS: squash root to %d:%d, except nid 0x%llx\n", - mds->mds_squash_uid, mds->mds_squash_gid, - mds->mds_nosquash_nid); - } - RETURN(0); - } - default: - CDEBUG(D_INFO, "unknown command %x\n", cmd); - RETURN(-EINVAL); + cfs_bitmap_set(mds->mds_lov_page_dirty, page); } - RETURN(0); - + CDEBUG(D_INFO, "idx "LPU64" - %p - %d/%d - "LPU64"\n", + idx, data, page, off, data[off]); +out: + RETURN(rc); } -struct mds_dt_sync_info { - struct obd_device *mlsi_obd; /* the mds to sync */ - struct obd_device *mlsi_watched; /* new lov target */ - int mlsi_index; /* index into mds_dt_objids */ -}; - -int mds_dt_synchronize(void *data) +int mds_lov_clear_orphans(struct mds_obd *mds, struct obd_uuid *ost_uuid) { - struct mds_dt_sync_info *mlsi = data; - struct llog_ctxt *ctxt; - struct obd_device *obd; - struct obd_device *watched; - struct mds_obd *mds; - struct obd_uuid *uuid; - obd_id vals[2]; - unsigned long flags; - __u32 vallen; - __u32 group; - int old_count; - int count; - int index; int rc; - char name[32] = "CATLIST"; - - lock_kernel(); - - ptlrpc_daemonize(); - - SIGNAL_MASK_LOCK(current, flags); - sigfillset(¤t->blocked); - RECALC_SIGPENDING; - SIGNAL_MASK_UNLOCK(current, flags); + struct obdo oa = { 0 }; + struct obd_trans_info oti = {0}; + struct lov_stripe_md *empty_ea = NULL; + ENTRY; - unlock_kernel(); + LASSERT(mds->mds_lov_page_array != NULL); - obd = mlsi->mlsi_obd; - watched = mlsi->mlsi_watched; - index = mlsi->mlsi_index; + /* This create will in fact either create or destroy: If the OST is + * missing objects below this ID, they will be created. If it finds + * objects above this ID, they will be removed. */ + memset(&oa, 0, sizeof(oa)); + oa.o_flags = OBD_FL_DELORPHAN; + oa.o_seq = mdt_to_obd_objseq(mds->mds_id); + oa.o_valid = OBD_MD_FLFLAGS | OBD_MD_FLGROUP; + if (ost_uuid != NULL) + oti.oti_ost_uuid = ost_uuid; - LASSERT(obd != NULL); - LASSERT(watched != NULL); + rc = obd_create(mds->mds_lov_exp, &oa, &empty_ea, &oti); - OBD_FREE(mlsi, sizeof(*mlsi)); + RETURN(rc); +} - mds = &obd->u.mds; - down(&mds->mds_orphan_recovery_sem); +/* for one target */ +static int mds_lov_set_one_nextid(struct obd_device *obd, __u32 idx, obd_id *id) +{ + struct mds_obd *mds = &obd->u.mds; + int rc; + struct obd_id_info info; + ENTRY; - uuid = &watched->u.cli.cl_import->imp_target_uuid; + LASSERT(!obd->obd_recovering); - group = FILTER_GROUP_FIRST_MDS + mds->mds_num; - rc = obd_set_info(watched->obd_self_export, strlen("mds_conn"), - "mds_conn", sizeof(group), &group); + info.idx = idx; + info.data = id; + rc = obd_set_info_async(mds->mds_lov_exp, sizeof(KEY_NEXT_ID), + KEY_NEXT_ID, sizeof(info), &info, NULL); if (rc) - GOTO(cleanup, rc); + CERROR ("%s: mds_lov_set_nextid failed (%d)\n", + obd->obd_name, rc); - old_count = mds->mds_dt_desc.ld_tgt_count; + RETURN(rc); +} - rc = mds_dt_update_desc(obd, mds->mds_dt_exp); - if (rc) - GOTO(cleanup, rc); +/* Update the lov desc for a new size lov. */ +static int mds_lov_update_desc(struct obd_device *obd, int idx, + struct obd_uuid *uuid) +{ + struct mds_obd *mds = &obd->u.mds; + struct lov_desc *ld; + __u32 valsize = sizeof(mds->mds_lov_desc); + int rc = 0; + ENTRY; - count = mds->mds_dt_desc.ld_tgt_count; - LASSERT(count >= old_count); + OBD_ALLOC(ld, sizeof(*ld)); + if (!ld) + RETURN(-ENOMEM); - vallen = sizeof(vals[1]); - rc = obd_get_info(watched->obd_self_export, strlen("last_id"), - "last_id", &vallen, &vals[1]); + rc = obd_get_info(mds->mds_lov_exp, sizeof(KEY_LOVDESC), KEY_LOVDESC, + &valsize, ld, NULL); if (rc) - GOTO(cleanup, rc); - - vals[0] = index; - rc = mds_dt_set_info(obd->obd_self_export, strlen("next_id"), - "next_id", 2, vals); + GOTO(out, rc); + + /* Don't change the mds_lov_desc until the objids size matches the + count (paranoia) */ + mds->mds_lov_desc = *ld; + CDEBUG(D_CONFIG, "updated lov_desc, tgt_count: %d - idx %d / uuid %s\n", + mds->mds_lov_desc.ld_tgt_count, idx, uuid->uuid); + + cfs_mutex_down(&obd->obd_dev_sem); + rc = mds_lov_update_max_ost(mds, idx); + cfs_mutex_up(&obd->obd_dev_sem); + if (rc != 0) + GOTO(out, rc ); + + /* If we added a target we have to reconnect the llogs */ + /* We only _need_ to do this at first add (idx), or the first time + after recovery. However, it should now be safe to call anytime. */ + rc = obd_llog_init(obd, &obd->obd_olg, obd, &idx); if (rc) - GOTO(cleanup, rc); + GOTO(out, rc); - obd_llog_finish(obd, &obd->obd_llogs, old_count); - obd_llog_cat_initialize(obd, &obd->obd_llogs, count, name); +out: + OBD_FREE(ld, sizeof(*ld)); + RETURN(rc); +} - ctxt = llog_get_context(&obd->obd_llogs, LLOG_UNLINK_ORIG_CTXT); - LASSERT(ctxt != NULL); +/* Inform MDS about new/updated target */ +static int mds_lov_update_mds(struct obd_device *obd, + struct obd_device *watched, + __u32 idx) +{ + struct mds_obd *mds = &obd->u.mds; + int rc = 0; + int page; + int off; + obd_id *data; + ENTRY; - rc = llog_connect(ctxt, count, NULL, NULL, uuid); - if (rc != 0) { - CERROR("%s: failed at llog_origin_connect: %d\n", - obd->obd_name, rc); - GOTO(cleanup, rc); - } - - CWARN("MDS %s: %s now active, resetting orphans\n", - obd->obd_name, uuid->uuid); + LASSERT(mds_lov_objinit(mds, idx)); - rc = mds_dt_clearorphans(&obd->u.mds, uuid); - if (rc != 0) { - CERROR("%s: failed at mds_dt_clearorphans(): %d\n", - obd->obd_name, rc); - GOTO(cleanup, rc); - } - rc = 0; + CDEBUG(D_CONFIG, "idx=%d, recov=%d/%d, cnt=%d\n", + idx, obd->obd_recovering, obd->obd_async_recov, + mds->mds_lov_desc.ld_tgt_count); - EXIT; -cleanup: - up(&mds->mds_orphan_recovery_sem); - return rc; -} + /* idx is set as data from lov_notify. */ + if (obd->obd_recovering) + GOTO(out, rc); -int mds_dt_start_synchronize(struct obd_device *obd, - struct obd_device *watched, - void *data) -{ - struct mds_dt_sync_info *mlsi; - int rc; - - ENTRY; + if (idx >= mds->mds_lov_desc.ld_tgt_count) { + CERROR("index %d > count %d!\n", idx, + mds->mds_lov_desc.ld_tgt_count); + GOTO(out, rc = -EINVAL); + } - OBD_ALLOC(mlsi, sizeof(*mlsi)); - if (mlsi == NULL) - RETURN(-ENOMEM); + rc = mds_lov_get_objid(obd, idx); + if (rc) + GOTO(out, rc); - mlsi->mlsi_obd = obd; - mlsi->mlsi_watched = watched; - mlsi->mlsi_index = (int)data; - - rc = kernel_thread(mds_dt_synchronize, mlsi, CLONE_VM | CLONE_FILES); - if (rc < 0) - CERROR("%s: error starting mds_dt_synchronize(): %d\n", - obd->obd_name, rc); - else { - CDEBUG(D_HA, "%s: mds_dt_synchronize() thread: %d\n", - obd->obd_name, rc); + page = idx / OBJID_PER_PAGE(); + off = idx % OBJID_PER_PAGE(); + data = mds->mds_lov_page_array[page]; + + /* We have read this lastid from disk; tell the osc. + Don't call this during recovery. */ + rc = mds_lov_set_one_nextid(obd, idx, &data[off]); + if (rc) { + CERROR("Failed to set next id, idx=%d rc=%d\n", idx,rc); + /* Don't abort the rest of the sync */ rc = 0; + } else { + CDEBUG(D_CONFIG, "last object "LPU64" from OST %d rc=%d\n", + data[off], idx, rc); } - +out: RETURN(rc); } -int mds_notify(struct obd_device *obd, struct obd_device *watched, - int active, void *data) +/* update the LOV-OSC knowledge of the last used object id's */ +int mds_lov_connect(struct obd_device *obd, char * lov_name) { - int rc = 0; + struct mds_obd *mds = &obd->u.mds; + struct obd_connect_data *data; + int rc; ENTRY; - if (!active) - RETURN(0); + if (IS_ERR(mds->mds_lov_obd)) + RETURN(PTR_ERR(mds->mds_lov_obd)); - if (!strcmp(watched->obd_type->typ_name, LUSTRE_MDC_NAME)) + if (mds->mds_lov_obd) RETURN(0); - if (strcmp(watched->obd_type->typ_name, LUSTRE_OSC_NAME)) { - CERROR("unexpected notification of %s %s!\n", - watched->obd_type->typ_name, watched->obd_name); - RETURN(-EINVAL); + mds->mds_lov_obd = class_name2obd(lov_name); + if (!mds->mds_lov_obd) { + CERROR("MDS cannot locate LOV %s\n", lov_name); + mds->mds_lov_obd = ERR_PTR(-ENOTCONN); + RETURN(-ENOTCONN); } - if (obd->obd_recovering) { - struct obd_uuid *uuid; - uuid = &watched->u.cli.cl_import->imp_target_uuid; + cfs_mutex_down(&obd->obd_dev_sem); + rc = mds_lov_read_objids(obd); + cfs_mutex_up(&obd->obd_dev_sem); + if (rc) { + CERROR("cannot read %s: rc = %d\n", "lov_objids", rc); + GOTO(err_exit, rc); + } - CWARN("MDS %s: in recovery, not resetting orphans on %s\n", - obd->obd_name, uuid->uuid); - } else { - rc = mds_dt_start_synchronize(obd, watched, data); + rc = obd_register_observer(mds->mds_lov_obd, obd); + if (rc) { + CERROR("MDS cannot register as observer of LOV %s (%d)\n", + lov_name, rc); + GOTO(err_exit, rc); } + + /* ask lov to generate OBD_NOTIFY_CREATE events for already registered + * targets */ + obd_notify(mds->mds_lov_obd, NULL, OBD_NOTIFY_CREATE, NULL); + + mds->mds_lov_obd->u.lov.lov_sp_me = LUSTRE_SP_MDT; + + OBD_ALLOC(data, sizeof(*data)); + if (data == NULL) + GOTO(err_exit, rc = -ENOMEM); + + data->ocd_connect_flags = OBD_CONNECT_VERSION | OBD_CONNECT_INDEX | + OBD_CONNECT_REQPORTAL | OBD_CONNECT_QUOTA64 | + OBD_CONNECT_OSS_CAPA | OBD_CONNECT_FID | + OBD_CONNECT_BRW_SIZE | OBD_CONNECT_CKSUM | + OBD_CONNECT_CHANGE_QS | OBD_CONNECT_AT | + OBD_CONNECT_MDS | OBD_CONNECT_SKIP_ORPHAN | + OBD_CONNECT_SOM | OBD_CONNECT_FULL20 | + OBD_CONNECT_64BITHASH; +#ifdef HAVE_LRU_RESIZE_SUPPORT + data->ocd_connect_flags |= OBD_CONNECT_LRU_RESIZE; +#endif + data->ocd_version = LUSTRE_VERSION_CODE; + data->ocd_group = mdt_to_obd_objseq(mds->mds_id); + /* send max bytes per rpc */ + data->ocd_brw_size = PTLRPC_MAX_BRW_PAGES << CFS_PAGE_SHIFT; + /* send the list of supported checksum types */ + data->ocd_cksum_types = cksum_types_supported(); + /* NB: lov_connect() needs to fill in .ocd_index for each OST */ + rc = obd_connect(NULL, &mds->mds_lov_exp, mds->mds_lov_obd, &obd->obd_uuid, data, NULL); + OBD_FREE(data, sizeof(*data)); + if (rc) { + CERROR("MDS cannot connect to LOV %s (%d)\n", lov_name, rc); + mds->mds_lov_obd = ERR_PTR(rc); + RETURN(rc); + } + + /* I want to see a callback happen when the OBD moves to a + * "For General Use" state, and that's when we'll call + * set_nextid(). The class driver can help us here, because + * it can use the obd_recovering flag to determine when the + * the OBD is full available. */ + /* MDD device will care about that + if (!obd->obd_recovering) + rc = mds_postrecov(obd); + */ + RETURN(rc); + +err_exit: + mds->mds_lov_exp = NULL; + mds->mds_lov_obd = ERR_PTR(rc); RETURN(rc); } -int mds_dt_set_info(struct obd_export *exp, obd_count keylen, - void *key, obd_count vallen, void *val) +int mds_lov_disconnect(struct obd_device *obd) { - struct obd_device *obd = class_exp2obd(exp); struct mds_obd *mds = &obd->u.mds; + int rc = 0; ENTRY; -#define KEY_IS(str) \ - (keylen == strlen(str) && memcmp(key, str, keylen) == 0) + if (!IS_ERR(mds->mds_lov_obd) && mds->mds_lov_exp != NULL) { + obd_register_observer(mds->mds_lov_obd, NULL); - if (KEY_IS("next_id")) { - obd_id *id = (obd_id *)val; - int idx; + /* The actual disconnect of the mds_lov will be called from + * class_disconnect_exports from mds_lov_clean. So we have to + * ensure that class_cleanup doesn't fail due to the extra ref + * we're holding now. The mechanism to do that already exists - + * the obd_force flag. We'll drop the final ref to the + * mds_lov_exp in mds_cleanup. */ + mds->mds_lov_obd->obd_force = 1; + } - /* XXX - this really should be vallen != (2 * sizeof(*id)) * - * Just following the precedent set by lov_set_info. */ - if (vallen != 2) - RETURN(-EINVAL); + RETURN(rc); +} - idx = *id; - if ((idx != *id) || (idx >= mds->mds_dt_desc.ld_tgt_count)) - RETURN(-EINVAL); +struct mds_lov_sync_info { + struct obd_device *mlsi_obd; /* the lov device to sync */ + struct obd_device *mlsi_watched; /* target osc */ + __u32 mlsi_index; /* index of target */ +}; - CDEBUG(D_CONFIG, "idx: %d id: %llu\n", idx, *(id + 1)); +static int mds_propagate_capa_keys(struct mds_obd *mds, struct obd_uuid *uuid) +{ + struct mds_capa_info info = { .uuid = uuid }; + struct lustre_capa_key *key; + int i, rc = 0; + + ENTRY; - mds->mds_dt_objids[idx] = *++id; - CDEBUG(D_CONFIG, "objid: %d: %lld\n", idx, *id); - /* XXX - should we be writing this out here ? */ - RETURN(mds_dt_write_objids(obd)); + if (!mds->mds_capa_keys) + RETURN(0); + + OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_SYNC_CAPA_SL, 5); + for (i = 0; i < 2; i++) { + key = &mds->mds_capa_keys[i]; + DEBUG_CAPA_KEY(D_SEC, key, "propagate"); + + info.capa = key; + rc = obd_set_info_async(mds->mds_lov_exp, sizeof(KEY_CAPA_KEY), + KEY_CAPA_KEY, sizeof(info), &info, NULL); + if (rc) { + DEBUG_CAPA_KEY(D_ERROR, key, + "propagate failed (rc = %d) for", rc); + RETURN(rc); + } } - RETURN(-EINVAL); + RETURN(0); } -int mds_dt_update_config(struct obd_device *obd, int clean) +/* We only sync one osc at a time, so that we don't have to hold + any kind of lock on the whole mds_lov_desc, which may change + (grow) as a result of mds_lov_add_ost. This also avoids any + kind of mismatch between the lov_desc and the mds_lov_desc, + which are not in lock-step during lov_add_obd */ +static int __mds_lov_synchronize(void *data) { + struct mds_lov_sync_info *mlsi = data; + struct obd_device *obd = mlsi->mlsi_obd; + struct obd_device *watched = mlsi->mlsi_watched; struct mds_obd *mds = &obd->u.mds; - struct lvfs_run_ctxt saved; - struct config_llog_instance cfg; + struct obd_uuid *uuid; + __u32 idx = mlsi->mlsi_index; + struct mds_group_info mgi; struct llog_ctxt *ctxt; - char *profile = mds->mds_profile, *name; - int rc, version, namelen; + int rc = 0; ENTRY; - if (profile == NULL) - RETURN(0); + OBD_FREE_PTR(mlsi); - cfg.cfg_instance = NULL; - cfg.cfg_uuid = mds->mds_dt_uuid; + LASSERT(obd); + LASSERT(watched); + uuid = &watched->u.cli.cl_target_uuid; + LASSERT(uuid); - namelen = strlen(profile) + 20; /* -clean-######### */ - OBD_ALLOC(name, namelen); - if (name == NULL) - RETURN(-ENOMEM); + cfs_down_read(&mds->mds_notify_lock); + if (obd->obd_stopping || obd->obd_fail) + GOTO(out, rc = -ENODEV); - if (clean) { - version = mds->mds_config_version - 1; - sprintf(name, "%s-clean-%d", profile, version); - } else { - version = mds->mds_config_version + 1; - sprintf(name, "%s-%d", profile, version); + OBD_RACE(OBD_FAIL_MDS_LOV_SYNC_RACE); + rc = mds_lov_update_mds(obd, watched, idx); + if (rc != 0) { + CERROR("%s failed at update_mds: %d\n", obd_uuid2str(uuid), rc); + GOTO(out, rc); } + mgi.group = mdt_to_obd_objseq(mds->mds_id); + mgi.uuid = uuid; + + rc = obd_set_info_async(mds->mds_lov_exp, sizeof(KEY_MDS_CONN), + KEY_MDS_CONN, sizeof(mgi), &mgi, NULL); + if (rc != 0) + GOTO(out, rc); + /* propagate capability keys */ + rc = mds_propagate_capa_keys(mds, uuid); + if (rc) + GOTO(out, rc); - CWARN("Applying configuration log %s\n", name); + ctxt = llog_get_context(obd, LLOG_MDS_OST_ORIG_CTXT); + if (!ctxt) + GOTO(out, rc = -ENODEV); - push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - ctxt = llog_get_context(&obd->obd_llogs, LLOG_CONFIG_ORIG_CTXT); - rc = class_config_process_llog(ctxt, name, &cfg); - pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - if (rc == 0) - mds->mds_config_version = version; - CWARN("Finished applying configuration log %s: %d\n", name, rc); + OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_LLOG_SYNC_TIMEOUT, 60); + rc = llog_connect(ctxt, NULL, NULL, uuid); + llog_ctxt_put(ctxt); + if (rc != 0) { + CERROR("%s failed at llog_origin_connect: %d\n", + obd_uuid2str(uuid), rc); + GOTO(out, rc); + } - OBD_FREE(name, namelen); - RETURN(rc); + LCONSOLE_INFO("MDS %s: %s now active, resetting orphans\n", + obd->obd_name, obd_uuid2str(uuid)); + + rc = mds_lov_clear_orphans(mds, uuid); + if (rc != 0) { + CERROR("%s failed at mds_lov_clear_orphans: %d\n", + obd_uuid2str(uuid), rc); + GOTO(out, rc); + } + +#ifdef HAVE_QUOTA_SUPPORT + if (obd->obd_upcall.onu_owner) { + /* + * This is a hack for mds_notify->mdd_notify. When the mds obd + * in mdd is removed, This hack should be removed. + */ + LASSERT(obd->obd_upcall.onu_upcall != NULL); + rc = obd->obd_upcall.onu_upcall(obd, NULL, OBD_NOTIFY_QUOTA, + obd->obd_upcall.onu_owner,NULL); + } +#endif + EXIT; +out: + cfs_up_read(&mds->mds_notify_lock); + if (rc) { + /* Deactivate it for safety */ + CERROR("%s sync failed %d, deactivating\n", obd_uuid2str(uuid), + rc); + if (!obd->obd_stopping && mds->mds_lov_obd && + !mds->mds_lov_obd->obd_stopping && !watched->obd_stopping) + obd_notify(mds->mds_lov_obd, watched, + OBD_NOTIFY_INACTIVE, NULL); + } + + class_decref(obd, "mds_lov_synchronize", obd); + return rc; } -/* Convert the on-disk LOV EA structre. - * We always try to convert from an old LOV EA format to the common in-memory - * (lsm) format (obd_unpackmd() understands the old on-disk (lmm) format) and - * then convert back to the new on-disk format and save it back to disk - * (obd_packmd() only ever saves to the new on-disk format) so we don't have - * to convert it each time this inode is accessed. - * - * This function is a bit interesting in the error handling. We can safely - * ship the old lmm to the client in case of failure, since it uses the same - * obd_unpackmd() code and can do the conversion if the MDS fails for some - * reason. We will not delete the old lmm data until we have written the - * new format lmm data in fsfilt_set_md(). */ -int mds_convert_lov_ea(struct obd_device *obd, struct inode *inode, - struct lov_mds_md *lmm, int lmm_size) +int mds_lov_synchronize(void *data) { - struct lov_stripe_md *lsm = NULL; - void *handle; - int rc, err; - ENTRY; + struct mds_lov_sync_info *mlsi = data; + char name[20]; - if (le32_to_cpu(lmm->lmm_magic) == LOV_MAGIC) - RETURN(0); + snprintf(name, sizeof(name), "ll_sync_%02u", mlsi->mlsi_index); + cfs_daemonize_ctxt(name); - CDEBUG(D_INODE, "converting LOV EA on %lu/%u from V0 to V1\n", - inode->i_ino, inode->i_generation); - rc = obd_unpackmd(obd->u.mds.mds_dt_exp, &lsm, lmm, lmm_size); - if (rc < 0) - GOTO(conv_end, rc); - - rc = obd_packmd(obd->u.mds.mds_dt_exp, &lmm, lsm); - if (rc < 0) - GOTO(conv_free, rc); - lmm_size = rc; - - handle = fsfilt_start(obd, inode, FSFILT_OP_SETATTR, NULL); - if (IS_ERR(handle)) { - rc = PTR_ERR(handle); - GOTO(conv_free, rc); - } - - rc = fsfilt_set_md(obd, inode, handle, lmm, lmm_size, EA_LOV); - err = fsfilt_commit(obd, obd->u.mds.mds_sb, inode, handle, 0); - if (!rc) - rc = err ? err : lmm_size; - GOTO(conv_free, rc); -conv_free: - obd_free_memmd(obd->u.mds.mds_dt_exp, &lsm); -conv_end: - return rc; + RETURN(__mds_lov_synchronize(data)); } -/* Must be called with i_sem held */ -int mds_revalidate_lov_ea(struct obd_device *obd, struct inode *inode, - struct lustre_msg *msg, int offset) +int mds_lov_start_synchronize(struct obd_device *obd, + struct obd_device *watched, + void *data, enum obd_notify_event ev) { - struct mds_obd *mds = &obd->u.mds; - struct obd_export *dt_exp = mds->mds_dt_exp; - struct lov_mds_md *lmm= NULL; - struct lov_stripe_md *lsm = NULL; - struct obdo *oa = NULL; - struct obd_trans_info oti = {0}; - obd_valid valid = 0; - int lmm_size = 0, lsm_size = 0, err, rc; - void *handle; + struct mds_lov_sync_info *mlsi; + int rc; + struct obd_uuid *uuid; ENTRY; - LASSERT(down_trylock(&inode->i_sem) != 0); + LASSERT(watched); + uuid = &watched->u.cli.cl_target_uuid; - /* XXX - add way to know if EA is already up to date & return - * without doing anything. Easy to do since we get notified of - * LOV updates. */ + OBD_ALLOC(mlsi, sizeof(*mlsi)); + if (mlsi == NULL) + RETURN(-ENOMEM); - lmm = lustre_msg_buf(msg, offset, 0); - if (lmm == NULL) { - CDEBUG(D_INFO, "no space reserved for inode %lu MD\n", - inode->i_ino); - RETURN(0); + LASSERT(data); + mlsi->mlsi_obd = obd; + mlsi->mlsi_watched = watched; + mlsi->mlsi_index = *(__u32 *)data; + + /* Although class_export_get(obd->obd_self_export) would lock + the MDS in place, since it's only a self-export + it doesn't lock the LOV in place. The LOV can be disconnected + during MDS precleanup, leaving nothing for __mds_lov_synchronize. + Simply taking an export ref on the LOV doesn't help, because it's + still disconnected. Taking an obd reference insures that we don't + disconnect the LOV. This of course means a cleanup won't + finish for as long as the sync is blocking. */ + class_incref(obd, "mds_lov_synchronize", obd); + + if (ev != OBD_NOTIFY_SYNC) { + /* Synchronize in the background */ + rc = cfs_create_thread(mds_lov_synchronize, mlsi, + CFS_DAEMON_FLAGS); + if (rc < 0) { + CERROR("%s: error starting mds_lov_synchronize: %d\n", + obd->obd_name, rc); + class_decref(obd, "mds_lov_synchronize", obd); + } else { + CDEBUG(D_HA, "%s: mds_lov_synchronize idx=%d " + "thread=%d\n", obd->obd_name, + mlsi->mlsi_index, rc); + rc = 0; + } + } else { + rc = __mds_lov_synchronize((void *)mlsi); } - lmm_size = msg->buflens[offset]; - - rc = obd_unpackmd(dt_exp, &lsm, lmm, lmm_size); - if (rc < 0) - RETURN(0); - - lsm_size = rc; - LASSERT(lsm->lsm_magic == LOV_MAGIC); + RETURN(rc); +} - oa = obdo_alloc(); - if (oa == NULL) - GOTO(out_lsm, rc = -ENOMEM); - oa->o_mode = S_IFREG | 0600; - oa->o_id = inode->i_ino; - oa->o_generation = inode->i_generation; - oa->o_uid = 0; /* must have 0 uid / gid on OST */ - oa->o_gid = 0; - oa->o_gr = FILTER_GROUP_FIRST_MDS + mds->mds_num; +int mds_notify(struct obd_device *obd, struct obd_device *watched, + enum obd_notify_event ev, void *data) +{ + struct mds_obd *mds = &obd->u.mds; + int rc = 0; + ENTRY; - oa->o_valid = OBD_MD_FLID | OBD_MD_FLGENER | OBD_MD_FLTYPE | - OBD_MD_FLMODE | OBD_MD_FLUID | OBD_MD_FLGID | - OBD_MD_FLGROUP; - valid = OBD_MD_FLTYPE | OBD_MD_FLATIME | OBD_MD_FLMTIME | - OBD_MD_FLCTIME; - obdo_from_inode(oa, inode, valid); + CDEBUG(D_CONFIG, "notify %s ev=%d\n", watched->obd_name, ev); - rc = obd_revalidate_md(dt_exp, oa, lsm, &oti); - if (rc == 0) - GOTO(out_oa, rc); - if (rc < 0) { - CERROR("Error validating LOV EA on %lu/%u: %d\n", - inode->i_ino, inode->i_generation, rc); - GOTO(out_oa, rc); + if (strcmp(watched->obd_type->typ_name, LUSTRE_OSC_NAME) != 0) { + CERROR("unexpected notification of %s %s!\n", + watched->obd_type->typ_name, watched->obd_name); + RETURN(-EINVAL); } - rc = obd_packmd(dt_exp, &lmm, lsm); - if (rc < 0) - GOTO(out_oa, rc); - lmm_size = rc; - - handle = fsfilt_start(obd, inode, FSFILT_OP_SETATTR, NULL); - if (IS_ERR(handle)) { - rc = PTR_ERR(handle); - GOTO(out_oa, rc); + /*XXX this notifies the MDD until lov handling use old mds code + * must non block! + */ + if (obd->obd_upcall.onu_owner) { + LASSERT(obd->obd_upcall.onu_upcall != NULL); + rc = obd->obd_upcall.onu_upcall(obd, NULL, ev, + obd->obd_upcall.onu_owner, + &mds->mds_obt.obt_mount_count); } - rc = fsfilt_set_md(obd, inode, handle, lmm, lmm_size, EA_LOV); - err = fsfilt_commit(obd, inode->i_sb, inode, handle, 0); - if (!rc) - rc = err; + switch (ev) { + /* We only handle these: */ + case OBD_NOTIFY_CREATE: + CWARN("MDS %s: add target %s\n",obd->obd_name, + obd_uuid2str(&watched->u.cli.cl_target_uuid)); + /* We still have to fix the lov descriptor for ost's */ + LASSERT(data); + rc = mds_lov_update_desc(obd, *(__u32 *)data, + &watched->u.cli.cl_target_uuid); + RETURN(rc); + case OBD_NOTIFY_ACTIVE: + /* lov want one or more _active_ targets for work */ + /* activate event should be pass lov idx as argument */ + case OBD_NOTIFY_SYNC: + case OBD_NOTIFY_SYNC_NONBLOCK: + /* sync event should be pass lov idx as argument */ + break; + default: + RETURN(0); + } - EXIT; -out_oa: - obdo_free(oa); -out_lsm: - obd_free_memmd(dt_exp, &lsm); - return rc; + if (obd->obd_recovering) { + CWARN("MDS %s: in recovery, not resetting orphans on %s\n", + obd->obd_name, + obd_uuid2str(&watched->u.cli.cl_target_uuid)); + /* We still have to fix the lov descriptor for ost's added + after the mdt in the config log. They didn't make it into + mds_lov_connect. */ + rc = mds_lov_update_desc(obd, *(__u32 *)data, + &watched->u.cli.cl_target_uuid); + } else { + rc = mds_lov_start_synchronize(obd, watched, data, ev); + } + RETURN(rc); }