* Copyright (C) 2001-2003 Cluster File Systems, Inc.
* Author: Peter Braam <braam@clusterfs.com>
*
- * This file is part of Lustre, http://www.lustre.org.
+ * This file is part of the Lustre file system, http://www.lustre.org
+ * Lustre is a trademark of Cluster File Systems, Inc.
*
- * Lustre is free software; you can redistribute it and/or
- * modify it under the terms of version 2 of the GNU General Public
- * License as published by the Free Software Foundation.
+ * You may have signed or agreed to another license before downloading
+ * this software. If so, you are bound by the terms and conditions
+ * of that agreement, and the following does not apply to you. See the
+ * LICENSE file included with this distribution for more information.
*
- * Lustre is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
+ * If you did not agree to a different license, then this copy of Lustre
+ * is open source software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
*
- * You should have received a copy of the GNU General Public License
- * along with Lustre; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ * In either case, Lustre is distributed in the hope that it will be
+ * useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * license text for more details.
*/
#ifndef EXPORT_SYMTAB
#define DEBUG_SUBSYSTEM S_MDS
#include <linux/module.h>
-#include <linux/lustre_mds.h>
-#include <linux/lustre_idl.h>
-#include <linux/obd_class.h>
-#include <linux/obd_lov.h>
-#include <linux/lustre_lib.h>
-#include <linux/lustre_fsfilt.h>
+#include <lustre_mds.h>
+#include <lustre/lustre_idl.h>
+#include <obd_class.h>
+#include <obd_lov.h>
+#include <lustre_lib.h>
+#include <lustre_fsfilt.h>
#include "mds_internal.h"
-void le_lov_desc_to_cpu (struct lov_desc *ld)
-{
- ld->ld_tgt_count = le32_to_cpu (ld->ld_tgt_count);
- ld->ld_default_stripe_count = le32_to_cpu (ld->ld_default_stripe_count);
- ld->ld_default_stripe_size = le32_to_cpu (ld->ld_default_stripe_size);
- ld->ld_pattern = le32_to_cpu (ld->ld_pattern);
-}
-
-void cpu_to_le_lov_desc (struct lov_desc *ld)
-{
- ld->ld_tgt_count = cpu_to_le32 (ld->ld_tgt_count);
- ld->ld_default_stripe_count = cpu_to_le32 (ld->ld_default_stripe_count);
- ld->ld_default_stripe_size = cpu_to_le32 (ld->ld_default_stripe_size);
- ld->ld_pattern = cpu_to_le32 (ld->ld_pattern);
-}
-
void mds_lov_update_objids(struct obd_device *obd, obd_id *ids)
{
struct mds_obd *mds = &obd->u.mds;
int i;
+ int dirty = 0;
ENTRY;
- lock_kernel();
- for (i = 0; i < mds->mds_lov_desc.ld_tgt_count; i++)
- if (ids[i] > (mds->mds_lov_objids)[i])
+ down_read(&mds->mds_lov_objids_sem);
+
+ spin_lock(&mds->mds_lov_objids_lock);
+ for (i = 0; i < mds->mds_lov_desc.ld_tgt_count; i++) {
+ if (ids[i] > (mds->mds_lov_objids)[i]) {
(mds->mds_lov_objids)[i] = ids[i];
- unlock_kernel();
+ dirty = 1;
+ }
+ }
+ mds->mds_lov_objids_dirty = dirty;
+
+ spin_unlock(&mds->mds_lov_objids_lock);
+
+ up_read(&mds->mds_lov_objids_sem);
+
EXIT;
}
+EXPORT_SYMBOL(mds_lov_update_objids);
static int mds_lov_read_objids(struct obd_device *obd)
{
struct mds_obd *mds = &obd->u.mds;
obd_id *ids;
loff_t off = 0;
- int i, rc, size = mds->mds_lov_desc.ld_tgt_count * sizeof(*ids);
+ int i, rc, size;
ENTRY;
- if (mds->mds_lov_objids != NULL)
- RETURN(0);
+ LASSERT(!mds->mds_lov_objids_count);
+ LASSERT(!mds->mds_lov_objids_dirty);
+
+ /* Read everything in the file, even if our current lov desc
+ has fewer targets. Old targets not in the lov descriptor
+ during mds setup may still have valid objids. */
+ size = i_size_read(mds->mds_lov_objid_filp->f_dentry->d_inode);
+ if (size == 0)
+ GOTO(out, rc = 0);
OBD_ALLOC(ids, size);
if (ids == NULL)
- RETURN(-ENOMEM);
- mds->mds_lov_objids = ids;
+ GOTO(out, rc = -ENOMEM);
- if (mds->mds_lov_objid_filp->f_dentry->d_inode->i_size == 0)
- RETURN(0);
rc = fsfilt_read_record(obd, mds->mds_lov_objid_filp, ids, size, &off);
if (rc < 0) {
+ OBD_FREE(ids, size);
CERROR("Error reading objids %d\n", rc);
- } else {
- mds->mds_lov_objids_valid = 1;
- rc = 0;
+ GOTO(out, rc);
}
+ mds->mds_lov_objids = ids;
+ mds->mds_lov_objids_count = size / sizeof(*ids);
- for (i = 0; i < mds->mds_lov_desc.ld_tgt_count; i++)
+ for (i = 0; i < mds->mds_lov_objids_count; i++) {
CDEBUG(D_INFO, "read last object "LPU64" for idx %d\n",
mds->mds_lov_objids[i], i);
+ }
+out:
RETURN(rc);
}
-
+/* must held mds_lov_objids_sem */
int mds_lov_write_objids(struct obd_device *obd)
{
struct mds_obd *mds = &obd->u.mds;
loff_t off = 0;
- int i, rc, size = mds->mds_lov_desc.ld_tgt_count * sizeof(obd_id);
+ int i, rc, tgts;
ENTRY;
+
+ spin_lock(&mds->mds_lov_objids_lock);
+ if (!mds->mds_lov_objids_dirty) {
+ spin_unlock(&mds->mds_lov_objids_lock);
+ RETURN(0);
+ }
+ mds->mds_lov_objids_dirty = 0;
+ spin_unlock(&mds->mds_lov_objids_lock);
- for (i = 0; i < mds->mds_lov_desc.ld_tgt_count; i++)
+ tgts = mds->mds_lov_objids_count;
+ if (!tgts)
+ RETURN(0);
+
+ for (i = 0; i < tgts; i++)
CDEBUG(D_INFO, "writing last object "LPU64" for idx %d\n",
mds->mds_lov_objids[i], i);
rc = fsfilt_write_record(obd, mds->mds_lov_objid_filp,
- mds->mds_lov_objids, size, &off, 0);
+ mds->mds_lov_objids, tgts * sizeof(obd_id),
+ &off, 0);
+ if (rc >= 0) {
+ rc = 0;
+ cfs_waitq_signal(&mds->mds_lov_objids_wait);
+ } else {
+ mds->mds_lov_objids_dirty = 1;
+ }
+
RETURN(rc);
}
+EXPORT_SYMBOL(mds_lov_write_objids);
-static int mds_lov_clearorphans(struct mds_obd *mds, struct obd_uuid *ost_uuid)
+int mds_lov_clear_orphans(struct mds_obd *mds, struct obd_uuid *ost_uuid)
{
int rc;
struct obdo oa;
* missing objects below this ID, they will be created. If it finds
* objects above this ID, they will be removed. */
memset(&oa, 0, sizeof(oa));
- oa.o_valid = OBD_MD_FLFLAGS;
oa.o_flags = OBD_FL_DELORPHAN;
+ oa.o_gr = FILTER_GROUP_MDS0 + mds->mds_id;
+ oa.o_valid = OBD_MD_FLFLAGS | OBD_MD_FLGROUP;
if (ost_uuid != NULL) {
memcpy(&oa.o_inline, ost_uuid, sizeof(*ost_uuid));
oa.o_valid |= OBD_MD_FLINLINE;
ENTRY;
LASSERT(!obd->obd_recovering);
+ if (mds->mds_lov_objids_count == 0)
+ RETURN(0);
LASSERT(mds->mds_lov_objids != NULL);
- rc = obd_set_info(mds->mds_osc_exp, strlen("next_id"), "next_id",
- mds->mds_lov_desc.ld_tgt_count, mds->mds_lov_objids);
- if (rc < 0)
- GOTO(out, rc);
+ /* mds->mds_lov_objids_sem must be held so mds_lov_objids doesn't change
+ * we need use mds->mds_lov_desc.ld_tgt_count because in recovery not all
+ * target can be connected at start time */
+ rc = obd_set_info_async(mds->mds_osc_exp, sizeof(KEY_NEXT_ID),
+ KEY_NEXT_ID,
+ mds->mds_lov_desc.ld_tgt_count *
+ sizeof(*mds->mds_lov_objids),
+ mds->mds_lov_objids, NULL);
- rc = mds_lov_clearorphans(mds, NULL /* all OSTs */);
- if (rc < 0)
+ if (rc)
+ CERROR ("%s: mds_lov_set_nextid failed (%d)\n",
+ obd->obd_name, rc);
+
+ RETURN(rc);
+}
+
+/* Update the lov desc for a new size lov. */
+static int mds_lov_update_desc(struct obd_device *obd, struct obd_export *lov)
+{
+ struct mds_obd *mds = &obd->u.mds;
+ struct lov_desc *ld;
+ __u32 size, stripes, valsize = sizeof(mds->mds_lov_desc);
+ int rc = 0;
+ ENTRY;
+
+ OBD_ALLOC(ld, sizeof(*ld));
+ if (!ld)
+ RETURN(-ENOMEM);
+
+ rc = obd_get_info(lov, sizeof(KEY_LOVDESC), KEY_LOVDESC,
+ &valsize, ld);
+ if (rc)
GOTO(out, rc);
-out:
- if (rc && mds->mds_lov_objids) {
- /* Might as well crash here, until we figure out what to do.
- * If we OBD_FREE, we'll just LASSERT the next time through this
- * function. */
- LBUG();
- OBD_FREE(mds->mds_lov_objids,
- mds->mds_lov_desc.ld_tgt_count * sizeof(obd_id));
- mds->mds_lov_objids = NULL;
+ /* The size of the LOV target table may have increased. */
+ if (ld->ld_tgt_count > mds->mds_lov_objids_count) {
+ obd_id *ids;
+
+ size = ld->ld_tgt_count * sizeof(obd_id);
+
+ OBD_ALLOC(ids, size);
+ if (ids == NULL)
+ GOTO(out, rc = -ENOMEM);
+ memset(ids, 0, size);
+
+ /* write lock is enough for protect from access to
+ * old pointer in mds_lov_update_objids and dirty == 0
+ * is enough for protect from access in write_objids */
+ if (mds->mds_lov_objids) {
+ struct l_wait_info lwi = { 0 };
+ obd_id *old_ids = mds->mds_lov_objids;
+
+ memcpy(ids, mds->mds_lov_objids,
+ mds_lov_objids_size(mds));
+
+ l_wait_event(mds->mds_lov_objids_wait,
+ mds->mds_lov_objids_dirty == 0, &lwi);
+
+ OBD_FREE(old_ids, mds_lov_objids_size(mds));
+ }
+ mds->mds_lov_objids = ids;
+ mds->mds_lov_objids_count = ld->ld_tgt_count;
}
+ /* Don't change the mds_lov_desc until the objids size matches the
+ count (paranoia) */
+ mds->mds_lov_desc = *ld;
+ CDEBUG(D_CONFIG, "updated lov_desc, tgt_count: %d\n",
+ mds->mds_lov_desc.ld_tgt_count);
+
+ stripes = min_t(__u32, LOV_MAX_STRIPE_COUNT,
+ max(mds->mds_lov_desc.ld_tgt_count,
+ mds->mds_lov_objids_count));
+ mds->mds_max_mdsize = lov_mds_md_size(stripes);
+ mds->mds_max_cookiesize = stripes * sizeof(struct llog_cookie);
+ CDEBUG(D_CONFIG, "updated max_mdsize/max_cookiesize for %d stripes: "
+ "%d/%d\n", mds->mds_max_mdsize, mds->mds_max_cookiesize,
+ stripes);
+
+ /* If we added a target we have to reconnect the llogs */
+ /* We only _need_ to do this at first add (idx), or the first time
+ after recovery. However, it should now be safe to call anytime. */
+ mutex_down(&obd->obd_dev_sem);
+ rc = llog_cat_initialize(obd, NULL, mds->mds_lov_desc.ld_tgt_count, NULL);
+ mutex_up(&obd->obd_dev_sem);
+
+ /*XXX this notifies the MDD until lov handling use old mds code */
+ if (obd->obd_upcall.onu_owner) {
+ LASSERT(obd->obd_upcall.onu_upcall != NULL);
+ rc = obd->obd_upcall.onu_upcall(NULL, NULL, 0,
+ obd->obd_upcall.onu_owner);
+ }
+out:
+ OBD_FREE(ld, sizeof(*ld));
RETURN(rc);
}
-/* tell the LOV-OSC by how much to pre-create */
-int mds_lov_set_growth(struct mds_obd *mds, int count)
+
+#define MDSLOV_NO_INDEX -1
+
+/* Inform MDS about new/updated target */
+static int mds_lov_update_mds(struct obd_device *obd,
+ struct obd_device *watched,
+ __u32 idx, struct obd_uuid *uuid)
{
- int rc;
+ struct mds_obd *mds = &obd->u.mds;
+ int old_count;
+ int rc = 0;
+ obd_id lastid;
+ __u32 size = sizeof(lastid);
+
ENTRY;
- rc = obd_set_info(mds->mds_osc_exp, strlen("growth_count"),
- "growth_count", sizeof(count), &count);
+ old_count = mds->mds_lov_desc.ld_tgt_count;
+ down_write(&mds->mds_lov_objids_sem);
+ rc = mds_lov_update_desc(obd, mds->mds_osc_exp);
+ downgrade_write(&mds->mds_lov_objids_sem);
+ if (rc)
+ GOTO(out, rc);
+
+ CDEBUG(D_CONFIG, "idx=%d, recov=%d/%d, cnt=%d/%d\n",
+ idx, obd->obd_recovering, obd->obd_async_recov, old_count,
+ mds->mds_lov_desc.ld_tgt_count);
+
+ /* idx is set as data from lov_notify. */
+ if (idx == MDSLOV_NO_INDEX || obd->obd_recovering)
+ GOTO(out, rc);
+
+ if (idx >= mds->mds_lov_desc.ld_tgt_count) {
+ CERROR("index %d > count %d!\n", idx,
+ mds->mds_lov_desc.ld_tgt_count);
+ GOTO(out, rc = -EINVAL);
+ }
+ rc = obd_get_info(watched->obd_self_export, sizeof("last_id"),
+ "last_id", &size, &lastid);
+ if (rc)
+ GOTO(out, rc);
+ spin_lock(&mds->mds_lov_objids_lock);
+ if (mds->mds_lov_objids[idx] == 0 || mds->mds_lov_objids[idx] > lastid) {
+ /* last id not init or corrupted - use data from osc */
+ mds->mds_lov_objids[idx] = lastid;
+ mds->mds_lov_objids_dirty = 1;
+ spin_unlock(&mds->mds_lov_objids_lock);
+ /* not need write immediately, mark for write for avoid
+ * lock inverse */
+ } else {
+ spin_unlock(&mds->mds_lov_objids_lock);
+ /* We have read this lastid from disk; tell the osc.
+ Don't call this during recovery. */
+ rc = mds_lov_set_nextid(obd);
+ }
+
+ CDEBUG(D_CONFIG, "last object "LPU64" from OST %d\n",
+ mds->mds_lov_objids[idx], idx);
+
+out:
+ up_read(&mds->mds_lov_objids_sem);
RETURN(rc);
}
+int mds_update_objids_from_lastid(struct obd_device *obd)
+{
+ struct mds_obd *mds = &obd->u.mds;
+ int size;
+ int rc, i;
+
+ if (mds->mds_lov_objids_count < mds->mds_lov_desc.ld_tgt_count) {
+ obd_id *ids;
+
+ size = mds->mds_lov_desc.ld_tgt_count * sizeof(obd_id);
+ OBD_ALLOC(ids, size);
+ if (ids == NULL)
+ GOTO(out, rc = -ENOMEM);
+
+ OBD_FREE(mds->mds_lov_objids, mds_lov_objids_size(mds));
+ mds->mds_lov_objids = ids;
+ mds->mds_lov_objids_count = size / sizeof(obd_id);
+ }
+
+ size = mds->mds_lov_desc.ld_tgt_count * sizeof(obd_id);
+ rc = obd_get_info(mds->mds_osc_exp, sizeof("last_id"),
+ "last_id", &size, mds->mds_lov_objids);
+ if (!rc) {
+ for (i = 0; i < mds->mds_lov_desc.ld_tgt_count; i++)
+ CWARN("got last object "LPU64" from OST %d\n",
+ mds->mds_lov_objids[i], i);
+ mds->mds_lov_objids_dirty = 1;
+ rc = mds_lov_write_objids(obd);
+ if (rc)
+ CERROR("got last objids from OSTs, but error "
+ "writing objids file: %d\n", rc);
+ }
+out:
+ RETURN(rc);
+}
+
+
+/* update the LOV-OSC knowledge of the last used object id's */
int mds_lov_connect(struct obd_device *obd, char * lov_name)
{
struct mds_obd *mds = &obd->u.mds;
struct lustre_handle conn = {0,};
- int valsize;
- int rc, i;
+ struct obd_connect_data *data;
+ int rc;
ENTRY;
if (IS_ERR(mds->mds_osc_obd))
mds->mds_osc_obd = class_name2obd(lov_name);
if (!mds->mds_osc_obd) {
- CERROR("MDS cannot locate LOV %s\n",
- lov_name);
+ CERROR("MDS cannot locate LOV %s\n", lov_name);
mds->mds_osc_obd = ERR_PTR(-ENOTCONN);
RETURN(-ENOTCONN);
}
- rc = obd_connect(&conn, mds->mds_osc_obd, &obd->obd_uuid);
+ OBD_ALLOC(data, sizeof(*data));
+ if (data == NULL)
+ RETURN(-ENOMEM);
+ data->ocd_connect_flags = OBD_CONNECT_VERSION | OBD_CONNECT_INDEX |
+ OBD_CONNECT_REQPORTAL | OBD_CONNECT_QUOTA64 |
+ OBD_CONNECT_OSS_CAPA;
+#ifdef HAVE_LRU_RESIZE_SUPPORT
+ data->ocd_connect_flags |= OBD_CONNECT_LRU_RESIZE;
+#endif
+ data->ocd_version = LUSTRE_VERSION_CODE;
+ data->ocd_group = mds->mds_id + FILTER_GROUP_MDS0;
+ /* NB: lov_connect() needs to fill in .ocd_index for each OST */
+ rc = obd_connect(NULL, &conn, mds->mds_osc_obd, &obd->obd_uuid, data);
+ OBD_FREE(data, sizeof(*data));
if (rc) {
- CERROR("MDS cannot connect to LOV %s (%d)\n",
- lov_name, rc);
+ CERROR("MDS cannot connect to LOV %s (%d)\n", lov_name, rc);
mds->mds_osc_obd = ERR_PTR(rc);
RETURN(rc);
}
GOTO(err_discon, rc);
}
- valsize = sizeof(mds->mds_lov_desc);
- rc = obd_get_info(mds->mds_osc_exp, strlen("lovdesc") + 1, "lovdesc",
- &valsize, &mds->mds_lov_desc);
- if (rc)
- GOTO(err_reg, rc);
+ /* Deny new client connections until we are sure we have some OSTs */
+ obd->obd_no_conn = 1;
- mds->mds_max_mdsize = lov_mds_md_size(mds->mds_lov_desc.ld_tgt_count);
- mds->mds_max_cookiesize = mds->mds_lov_desc.ld_tgt_count*
- sizeof(struct llog_cookie);
- mds->mds_has_lov_desc = 1;
+ down_write(&mds->mds_lov_objids_sem);
rc = mds_lov_read_objids(obd);
if (rc) {
CERROR("cannot read %s: rc = %d\n", "lov_objids", rc);
GOTO(err_reg, rc);
- }
-
-#ifdef ENABLE_ORPHANS
- rc = llog_cat_initialize(obd, mds->mds_lov_desc.ld_tgt_count);
- if (rc) {
- CERROR("failed to initialize catalog %d\n", rc);
- GOTO(err_reg, rc);
}
-#endif
- /* FIXME before this set info call is made, we must initialize the logging */
- rc = obd_set_info(mds->mds_osc_exp, strlen("mds_conn"), "mds_conn",
- 0, NULL);
- if (rc)
+
+ rc = mds_lov_update_desc(obd, mds->mds_osc_exp);
+ if (rc)
GOTO(err_reg, rc);
-
+
/* If we're mounting this code for the first time on an existing FS,
* we need to populate the objids array from the real OST values */
- if (!mds->mds_lov_objids_valid) {
- int size = sizeof(obd_id) * mds->mds_lov_desc.ld_tgt_count;
- rc = obd_get_info(mds->mds_osc_exp, strlen("last_id"),
- "last_id", &size, mds->mds_lov_objids);
- if (!rc) {
- for (i = 0; i < mds->mds_lov_desc.ld_tgt_count; i++)
- CWARN("got last object "LPU64" from OST %d\n",
- mds->mds_lov_objids[i], i);
- mds->mds_lov_objids_valid = 1;
- rc = mds_lov_write_objids(obd);
- if (rc)
- CERROR("got last objids from OSTs, but error "
- "writing objids file: %d\n", rc);
- }
+ if (mds->mds_lov_desc.ld_tgt_count > mds->mds_lov_objids_count) {
+ mds_update_objids_from_lastid(obd);
}
+ up_write(&mds->mds_lov_objids_sem);
/* I want to see a callback happen when the OBD moves to a
* "For General Use" state, and that's when we'll call
* set_nextid(). The class driver can help us here, because
* it can use the obd_recovering flag to determine when the
* the OBD is full available. */
- if (!obd->obd_recovering) {
-#ifdef ENABLE_ORPHANS
- rc = llog_connect(llog_get_context(obd, LLOG_UNLINK_ORIG_CTXT),
- obd->u.mds.mds_lov_desc.ld_tgt_count, NULL, NULL);
- if (rc != 0) {
- CERROR("faild at llog_origin_connect: %d\n", rc);
- }
-#endif
- rc = mds_cleanup_orphans(obd);
- if (rc > 0)
- CERROR("Cleanup %d orphans while MDS isn't recovering\n", rc);
-
- rc = mds_lov_set_nextid(obd);
- if (rc)
- GOTO(err_llog, rc);
- }
+ /* MDD device will care about that
+ if (!obd->obd_recovering)
+ rc = mds_postrecov(obd);
+ */
RETURN(rc);
-err_llog:
-#ifdef ENABLE_ORPHANS
- /* cleanup all llogging subsystems */
- rc = obd_llog_finish(obd, mds->mds_lov_desc.ld_tgt_count);
- if (rc)
- CERROR("failed to cleanup llogging subsystems\n");
-#endif
err_reg:
+ up_write(&mds->mds_lov_objids_sem);
obd_register_observer(mds->mds_osc_obd, NULL);
err_discon:
- obd_disconnect(mds->mds_osc_exp, 0);
+ obd_disconnect(mds->mds_osc_exp);
mds->mds_osc_exp = NULL;
mds->mds_osc_obd = ERR_PTR(rc);
RETURN(rc);
}
-int mds_lov_disconnect(struct obd_device *obd, int flags)
+int mds_lov_disconnect(struct obd_device *obd)
{
struct mds_obd *mds = &obd->u.mds;
int rc = 0;
ENTRY;
if (!IS_ERR(mds->mds_osc_obd) && mds->mds_osc_exp != NULL) {
-#ifdef ENABLE_ORPHANS
- /* cleanup all llogging subsystems */
- rc = obd_llog_finish(obd, mds->mds_lov_desc.ld_tgt_count);
- if (rc)
- CERROR("failed to cleanup llogging subsystems\n");
-#endif
-
obd_register_observer(mds->mds_osc_obd, NULL);
- rc = obd_disconnect(mds->mds_osc_exp, flags);
- /* if obd_disconnect fails (probably because the
- * export was disconnected by class_disconnect_exports)
- * then we just need to drop our ref. */
- if (rc != 0)
- class_export_put(mds->mds_osc_exp);
- mds->mds_osc_exp = NULL;
- mds->mds_osc_obd = NULL;
+ /* The actual disconnect of the mds_lov will be called from
+ * class_disconnect_exports from mds_lov_clean. So we have to
+ * ensure that class_cleanup doesn't fail due to the extra ref
+ * we're holding now. The mechanism to do that already exists -
+ * the obd_force flag. We'll drop the final ref to the
+ * mds_osc_exp in mds_cleanup. */
+ mds->mds_osc_obd->obd_force = 1;
}
RETURN(rc);
struct obd_device *obd = exp->exp_obd;
struct mds_obd *mds = &obd->u.mds;
struct obd_ioctl_data *data = karg;
- struct obd_run_ctxt saved;
+ struct lvfs_run_ctxt saved;
int rc = 0;
+ ENTRY;
+ CDEBUG(D_IOCTL, "handling ioctl cmd %#x\n", cmd);
+
switch (cmd) {
case OBD_IOC_RECORD: {
char *name = data->ioc_inlbuf1;
if (mds->mds_cfg_llh)
RETURN(-EBUSY);
- push_ctxt(&saved, &obd->obd_ctxt, NULL);
- rc = llog_create(llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT),
+ push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
+ rc = llog_create(llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT),
&mds->mds_cfg_llh, NULL, name);
if (rc == 0)
- llog_init_handle(mds->mds_cfg_llh, LLOG_F_IS_PLAIN,
+ llog_init_handle(mds->mds_cfg_llh, LLOG_F_IS_PLAIN,
&cfg_uuid);
else
mds->mds_cfg_llh = NULL;
- pop_ctxt(&saved, &obd->obd_ctxt, NULL);
+ pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
RETURN(rc);
}
if (!mds->mds_cfg_llh)
RETURN(-EBADF);
- push_ctxt(&saved, &obd->obd_ctxt, NULL);
+ push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
rc = llog_close(mds->mds_cfg_llh);
- pop_ctxt(&saved, &obd->obd_ctxt, NULL);
+ pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
+
+ mds->mds_cfg_llh = NULL;
+ RETURN(rc);
+ }
+
+ case OBD_IOC_CLEAR_LOG: {
+ char *name = data->ioc_inlbuf1;
+ if (mds->mds_cfg_llh)
+ RETURN(-EBUSY);
+
+ push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
+ rc = llog_create(llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT),
+ &mds->mds_cfg_llh, NULL, name);
+ if (rc == 0) {
+ llog_init_handle(mds->mds_cfg_llh, LLOG_F_IS_PLAIN,
+ NULL);
+
+ rc = llog_destroy(mds->mds_cfg_llh);
+ llog_free_handle(mds->mds_cfg_llh);
+ }
+ pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
mds->mds_cfg_llh = NULL;
RETURN(rc);
if (data->ioc_type == LUSTRE_CFG_TYPE) {
rec.lrh_type = OBD_CFG_REC;
- } else if (data->ioc_type == PORTALS_CFG_TYPE) {
- rec.lrh_type = PTL_CFG_REC;
} else {
CERROR("unknown cfg record type:%d \n", data->ioc_type);
RETURN(-EINVAL);
RETURN(rc);
}
- push_ctxt(&saved, &obd->obd_ctxt, NULL);
+ push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
rc = llog_write_rec(mds->mds_cfg_llh, &rec, NULL, 0,
cfg_buf, -1);
- pop_ctxt(&saved, &obd->obd_ctxt, NULL);
+ pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
OBD_FREE(cfg_buf, data->ioc_plen1);
RETURN(rc);
}
case OBD_IOC_PARSE: {
- struct llog_ctxt *ctxt =
+ struct llog_ctxt *ctxt =
llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT);
- push_ctxt(&saved, &obd->obd_ctxt, NULL);
+ push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
rc = class_config_parse_llog(ctxt, data->ioc_inlbuf1, NULL);
- pop_ctxt(&saved, &obd->obd_ctxt, NULL);
+ pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
if (rc)
RETURN(rc);
}
case OBD_IOC_DUMP_LOG: {
- struct llog_ctxt *ctxt =
+ struct llog_ctxt *ctxt =
llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT);
- push_ctxt(&saved, &obd->obd_ctxt, NULL);
+ push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
rc = class_config_dump_llog(ctxt, data->ioc_inlbuf1, NULL);
- pop_ctxt(&saved, &obd->obd_ctxt, NULL);
+ pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL);
if (rc)
RETURN(rc);
RETURN(rc);
}
+ case OBD_IOC_SYNC: {
+ CDEBUG(D_HA, "syncing mds %s\n", obd->obd_name);
+ rc = fsfilt_sync(obd, obd->u.obt.obt_sb);
+ RETURN(rc);
+ }
+
case OBD_IOC_SET_READONLY: {
void *handle;
- struct inode *inode = obd->u.mds.mds_sb->s_root->d_inode;
+ struct inode *inode = obd->u.obt.obt_sb->s_root->d_inode;
BDEVNAME_DECLARE_STORAGE(tmp);
- CERROR("setting device %s read-only\n",
- ll_bdevname(obd->u.mds.mds_sb, tmp));
+ CERROR("*** setting device %s read-only ***\n",
+ ll_bdevname(obd->u.obt.obt_sb, tmp));
handle = fsfilt_start(obd, inode, FSFILT_OP_MKNOD, NULL);
- LASSERT(handle);
- rc = fsfilt_commit(obd, inode, handle, 1);
+ if (!IS_ERR(handle))
+ rc = fsfilt_commit(obd, inode, handle, 1);
- dev_set_rdonly(ll_sbdev(obd->u.mds.mds_sb), 2);
+ CDEBUG(D_HA, "syncing mds %s\n", obd->obd_name);
+ rc = fsfilt_sync(obd, obd->u.obt.obt_sb);
+
+ lvfs_set_rdonly(obd, obd->u.obt.obt_sb);
RETURN(0);
}
case OBD_IOC_CATLOGLIST: {
int count = mds->mds_lov_desc.ld_tgt_count;
- rc = llog_catlog_list(obd, count, data);
+ rc = llog_catalog_list(obd, count, data);
RETURN(rc);
}
- case OBD_IOC_LLOG_CHECK:
+ case OBD_IOC_LLOG_CHECK:
case OBD_IOC_LLOG_CANCEL:
- case OBD_IOC_LLOG_REMOVE: {
- struct llog_ctxt *ctxt =
+ case OBD_IOC_LLOG_REMOVE: {
+ struct llog_ctxt *ctxt =
llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT);
+ int rc2;
+ __u32 group;
-#ifdef ENABLE_ORPHANS
obd_llog_finish(obd, mds->mds_lov_desc.ld_tgt_count);
-#endif
- push_ctxt(&saved, &ctxt->loc_exp->exp_obd->obd_ctxt, NULL);
+ push_ctxt(&saved, &ctxt->loc_exp->exp_obd->obd_lvfs_ctxt, NULL);
rc = llog_ioctl(ctxt, cmd, data);
- pop_ctxt(&saved, &ctxt->loc_exp->exp_obd->obd_ctxt, NULL);
-
-#ifdef ENABLE_ORPHANS
- llog_cat_initialize(obd, mds->mds_lov_desc.ld_tgt_count);
-#endif
+ pop_ctxt(&saved, &ctxt->loc_exp->exp_obd->obd_lvfs_ctxt, NULL);
+ llog_cat_initialize(obd, NULL, mds->mds_lov_desc.ld_tgt_count, NULL);
+ group = FILTER_GROUP_MDS0 + mds->mds_id;
+ rc2 = obd_set_info_async(mds->mds_osc_exp,
+ strlen(KEY_MDS_CONN), KEY_MDS_CONN,
+ sizeof(group), &group, NULL);
+ if (!rc)
+ rc = rc2;
RETURN(rc);
- }
+ }
case OBD_IOC_LLOG_INFO:
case OBD_IOC_LLOG_PRINT: {
- struct llog_ctxt *ctxt =
+ struct llog_ctxt *ctxt =
llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT);
-
- push_ctxt(&saved, &ctxt->loc_exp->exp_obd->obd_ctxt, NULL);
+
+ push_ctxt(&saved, &ctxt->loc_exp->exp_obd->obd_lvfs_ctxt, NULL);
rc = llog_ioctl(ctxt, cmd, data);
- pop_ctxt(&saved, &ctxt->loc_exp->exp_obd->obd_ctxt, NULL);
-
+ pop_ctxt(&saved, &ctxt->loc_exp->exp_obd->obd_lvfs_ctxt, NULL);
+
RETURN(rc);
}
case OBD_IOC_ABORT_RECOVERY:
CERROR("aborting recovery for device %s\n", obd->obd_name);
- target_abort_recovery(obd);
+ target_stop_recovery_thread(obd);
RETURN(0);
default:
+ CDEBUG(D_INFO, "unknown command %x\n", cmd);
RETURN(-EINVAL);
}
RETURN(0);
+
+}
+
+struct mds_lov_sync_info {
+ struct obd_device *mlsi_obd; /* the lov device to sync */
+ struct obd_device *mlsi_watched; /* target osc */
+ __u32 mlsi_index; /* index of target */
+};
+
+static int mds_propagate_capa_keys(struct mds_obd *mds)
+{
+ struct lustre_capa_key *key;
+ int i, rc = 0;
+
+ ENTRY;
+
+ if (!mds->mds_capa_keys)
+ RETURN(0);
+
+ for (i = 0; i < 2; i++) {
+ key = &mds->mds_capa_keys[i];
+ DEBUG_CAPA_KEY(D_SEC, key, "propagate");
+
+ rc = obd_set_info_async(mds->mds_osc_exp, strlen(KEY_CAPA_KEY),
+ KEY_CAPA_KEY, sizeof(*key), key, NULL);
+ if (rc) {
+ DEBUG_CAPA_KEY(D_ERROR, key,
+ "propagate failed (rc = %d) for", rc);
+ RETURN(rc);
+ }
+ }
+
+ RETURN(0);
+}
+
+/* We only sync one osc at a time, so that we don't have to hold
+ any kind of lock on the whole mds_lov_desc, which may change
+ (grow) as a result of mds_lov_add_ost. This also avoids any
+ kind of mismatch between the lov_desc and the mds_lov_desc,
+ which are not in lock-step during lov_add_obd */
+static int __mds_lov_synchronize(void *data)
+{
+ struct mds_lov_sync_info *mlsi = data;
+ struct obd_device *obd = mlsi->mlsi_obd;
+ struct obd_device *watched = mlsi->mlsi_watched;
+ struct mds_obd *mds = &obd->u.mds;
+ struct obd_uuid *uuid;
+ __u32 idx = mlsi->mlsi_index;
+ struct mds_group_info mgi;
+ int rc = 0;
+ ENTRY;
+
+ OBD_FREE(mlsi, sizeof(*mlsi));
+
+ LASSERT(obd);
+ LASSERT(watched);
+ uuid = &watched->u.cli.cl_target_uuid;
+ LASSERT(uuid);
+
+ rc = mds_lov_update_mds(obd, watched, idx, uuid);
+ if (rc != 0)
+ GOTO(out, rc);
+ mgi.group = FILTER_GROUP_MDS0 + mds->mds_id;
+ mgi.uuid = uuid;
+ rc = obd_set_info_async(mds->mds_osc_exp, strlen(KEY_MDS_CONN),
+ KEY_MDS_CONN, sizeof(mgi), &mgi, NULL);
+ if (rc != 0)
+ GOTO(out, rc);
+
+ /* propagate capability keys */
+ rc = mds_propagate_capa_keys(mds);
+ if (rc)
+ GOTO(out, rc);
+
+ rc = llog_connect(llog_get_context(obd, LLOG_MDS_OST_ORIG_CTXT),
+ mds->mds_lov_desc.ld_tgt_count,
+ NULL, NULL, uuid);
+
+ if (rc != 0) {
+ CERROR("%s: failed at llog_origin_connect: %d\n",
+ obd->obd_name, rc);
+ GOTO(out, rc);
+ }
+
+ LCONSOLE_INFO("MDS %s: %s now active, resetting orphans\n",
+ obd->obd_name, obd_uuid2str(uuid));
+ /*
+ * FIXME: this obd_stopping was useless,
+ * since obd in mdt layer was set
+ */
+ if (obd->obd_stopping)
+ GOTO(out, rc = -ENODEV);
+
+ rc = mds_lov_clear_orphans(mds, uuid);
+ if (rc != 0) {
+ CERROR("%s: failed at mds_lov_clear_orphans: %d\n",
+ obd->obd_name, rc);
+ GOTO(out, rc);
+ }
+
+ if (obd->obd_upcall.onu_owner) {
+ /*
+ * This is a hack for mds_notify->mdd_notify. When the mds obd
+ * in mdd is removed, This hack should be removed.
+ */
+ LASSERT(obd->obd_upcall.onu_upcall != NULL);
+ rc = obd->obd_upcall.onu_upcall(NULL, NULL, 0,
+ obd->obd_upcall.onu_owner);
+ }
+ EXIT;
+out:
+ class_decref(obd);
+ return rc;
+}
+
+int mds_lov_synchronize(void *data)
+{
+ struct mds_lov_sync_info *mlsi = data;
+ char name[20];
+
+ if (mlsi->mlsi_index == MDSLOV_NO_INDEX)
+ /* There is still a watched target,
+ but we don't know its index */
+ sprintf(name, "ll_sync_tgt");
+ else
+ snprintf(name, sizeof(name), "ll_sync_%02u", mlsi->mlsi_index);
+ ptlrpc_daemonize(name);
+
+ RETURN(__mds_lov_synchronize(data));
+}
+
+int mds_lov_start_synchronize(struct obd_device *obd,
+ struct obd_device *watched,
+ void *data, int nonblock)
+{
+ struct mds_lov_sync_info *mlsi;
+ int rc;
+
+ ENTRY;
+
+ LASSERT(watched);
+
+ OBD_ALLOC(mlsi, sizeof(*mlsi));
+ if (mlsi == NULL)
+ RETURN(-ENOMEM);
+
+ mlsi->mlsi_obd = obd;
+ mlsi->mlsi_watched = watched;
+ if (data)
+ mlsi->mlsi_index = *(__u32 *)data;
+ else
+ mlsi->mlsi_index = MDSLOV_NO_INDEX;
+
+ /* Although class_export_get(obd->obd_self_export) would lock
+ the MDS in place, since it's only a self-export
+ it doesn't lock the LOV in place. The LOV can be disconnected
+ during MDS precleanup, leaving nothing for __mds_lov_synchronize.
+ Simply taking an export ref on the LOV doesn't help, because it's
+ still disconnected. Taking an obd reference insures that we don't
+ disconnect the LOV. This of course means a cleanup won't
+ finish for as long as the sync is blocking. */
+ class_incref(obd);
+
+ if (nonblock) {
+ /* Synchronize in the background */
+ rc = cfs_kernel_thread(mds_lov_synchronize, mlsi,
+ CLONE_VM | CLONE_FILES);
+ if (rc < 0) {
+ CERROR("%s: error starting mds_lov_synchronize: %d\n",
+ obd->obd_name, rc);
+ class_decref(obd);
+ } else {
+ CDEBUG(D_HA, "%s: mds_lov_synchronize idx=%d "
+ "thread=%d\n", obd->obd_name,
+ mlsi->mlsi_index, rc);
+ rc = 0;
+ }
+ } else {
+ rc = __mds_lov_synchronize((void *)mlsi);
+ }
+
+ RETURN(rc);
}
int mds_notify(struct obd_device *obd, struct obd_device *watched,
- int active)
+ enum obd_notify_event ev, void *data)
{
- struct obd_uuid *uuid;
+ struct mds_obd *mds = &obd->u.mds;
+ int rc = 0;
+ ENTRY;
- if (!active)
+ switch (ev) {
+ /* We only handle these: */
+ case OBD_NOTIFY_ACTIVE:
+ case OBD_NOTIFY_SYNC:
+ case OBD_NOTIFY_SYNC_NONBLOCK:
+ break;
+ case OBD_NOTIFY_CONFIG:
+ /* Open for clients */
+ obd->obd_no_conn = 0;
+ default:
RETURN(0);
+ }
- if (strcmp(watched->obd_type->typ_name, "osc")) {
+ CDEBUG(D_CONFIG, "notify %s ev=%d\n", watched->obd_name, ev);
+ if (strcmp(watched->obd_type->typ_name, LUSTRE_OSC_NAME) != 0) {
CERROR("unexpected notification of %s %s!\n",
- watched->obd_type->typ_name,
- watched->obd_name);
+ watched->obd_type->typ_name, watched->obd_name);
RETURN(-EINVAL);
}
- uuid = &watched->u.cli.cl_import->imp_target_uuid;
- CERROR("MDS %s: %s now active, resetting orphans\n",
- obd->obd_name, uuid->uuid);
- RETURN(mds_lov_clearorphans(&obd->u.mds, uuid));
+ if (obd->obd_recovering) {
+ CWARN("MDS %s: in recovery, not resetting orphans on %s\n",
+ obd->obd_name,
+ obd_uuid2str(&watched->u.cli.cl_target_uuid));
+ /* We still have to fix the lov descriptor for ost's added
+ after the mdt in the config log. They didn't make it into
+ mds_lov_connect. */
+ down_write(&mds->mds_lov_objids_sem);
+ rc = mds_lov_update_desc(obd, mds->mds_osc_exp);
+ up_write(&mds->mds_lov_objids_sem);
+ if (rc)
+ RETURN(rc);
+ /* We should update init llog here too for replay unlink and
+ * possiable llog init race when recovery complete */
+ mutex_down(&obd->obd_dev_sem);
+ llog_cat_initialize(obd, NULL,
+ obd->u.mds.mds_lov_desc.ld_tgt_count,
+ &watched->u.cli.cl_target_uuid);
+ mutex_up(&obd->obd_dev_sem);
+ RETURN(rc);
+ }
+
+ LASSERT(llog_get_context(obd, LLOG_MDS_OST_ORIG_CTXT) != NULL);
+ rc = mds_lov_start_synchronize(obd, watched, data,
+ !(ev == OBD_NOTIFY_SYNC));
+
+ lquota_recovery(mds_quota_interface_ref, obd);
+
+ RETURN(rc);
}
/* Convert the on-disk LOV EA structre.
int rc, err;
ENTRY;
- if (le32_to_cpu(lmm->lmm_magic) == LOV_MAGIC)
+ if (le32_to_cpu(lmm->lmm_magic) == LOV_MAGIC ||
+ le32_to_cpu(lmm->lmm_magic == LOV_MAGIC_JOIN))
RETURN(0);
- CWARN("converting LOV EA on %lu/%u from V0 to V1\n",
- inode->i_ino, inode->i_generation);
+ CDEBUG(D_INODE, "converting LOV EA on %lu/%u from %#08x to %#08x\n",
+ inode->i_ino, inode->i_generation, le32_to_cpu(lmm->lmm_magic),
+ LOV_MAGIC);
+
rc = obd_unpackmd(obd->u.mds.mds_osc_exp, &lsm, lmm, lmm_size);
if (rc < 0)
GOTO(conv_end, rc);
GOTO(conv_free, rc);
}
- rc = fsfilt_set_md(obd, inode, handle, lmm, lmm_size);
+ rc = fsfilt_set_md(obd, inode, handle, lmm, lmm_size, "lov");
err = fsfilt_commit(obd, inode, handle, 0);
if (!rc)
conv_end:
return rc;
}
+
+void mds_objids_from_lmm(obd_id *ids, struct lov_mds_md *lmm,
+ struct lov_desc *desc)
+{
+ int i;
+ for (i = 0; i < le32_to_cpu(lmm->lmm_stripe_count); i++) {
+ ids[le32_to_cpu(lmm->lmm_objects[i].l_ost_idx)] =
+ le64_to_cpu(lmm->lmm_objects[i].l_object_id);
+ }
+}
+EXPORT_SYMBOL(mds_objids_from_lmm);
git://git.whamcloud.com / fs / lustre-release.git / blobdiff