From 6b5f7efed3bd2b5102efd5f7d9604e12c1715f94 Mon Sep 17 00:00:00 2001 From: nathan Date: Thu, 2 Feb 2006 01:39:37 +0000 Subject: [PATCH] Branch b1_4_mountconf b=4482 Problem with the mds notify method of online ost addition - lov_notify can't notify the mds until the mds is set up, due to a check of obd_set_up in obd_notify. But obd_set_up isn't set until mds_postrecov/mds_postsetup/mds_setup returns to class_setup. (This can be seen by adding ost's before adding the mdt - the other order doesn't have the problem because there are no osc's in the lov during mds_postrecov at that point, so no notifies.) So I hacked in an additional flag check to obd_notify that happens only to be set for the mds at this point in it's life: obd_async_recov. Don't hurt me. --- lustre/include/linux/lustre_disk.h | 6 ++-- lustre/include/linux/lustre_idl.h | 4 ++- lustre/include/linux/obd_class.h | 11 +++++-- lustre/mds/handler.c | 3 +- lustre/mds/mds_lov.c | 5 +-- lustre/utils/mkfs_lustre.c | 67 +------------------------------------- 6 files changed, 20 insertions(+), 76 deletions(-) diff --git a/lustre/include/linux/lustre_disk.h b/lustre/include/linux/lustre_disk.h index d751c6b..3dec274 100644 --- a/lustre/include/linux/lustre_disk.h +++ b/lustre/include/linux/lustre_disk.h @@ -69,7 +69,7 @@ static inline char *mt_str(enum ldd_mount_type mt) } #ifndef MTI_NIDS_MAX /* match lustre_idl.h */ -#define MTI_NIDS_MAX 10 +#define MTI_NIDS_MAX 64 #endif #define LDD_INCOMPAT_SUPP 0 @@ -91,7 +91,7 @@ struct lustre_disk_data { svname */ __u16 ldd_mgsnid_count; __u16 ldd_failnid_count; /* server failover nid count */ - lnet_nid_t ldd_mgsnid[MTI_NIDS_MAX]; /* mgmt nid list; lmd can + lnet_nid_t ldd_mgsnid[MTI_NIDS_MAX]; /* mgs nid list; lmd can override */ lnet_nid_t ldd_failnid[MTI_NIDS_MAX]; /* server failover nids */ char ldd_mount_opts[2048]; /* target fs mount opts */ @@ -119,7 +119,7 @@ static inline int sv_make_name(__u32 flags, __u16 index, char *fs, char *name) (flags & LDD_F_SV_TYPE_MDT) ? "MDT" : "OST", index); } else if (flags & LDD_F_SV_TYPE_MGS) { - sprintf(name, "MGMT"); + sprintf(name, "MGS"); } else { CERROR("unknown server type %#x\n", flags); return 1; diff --git a/lustre/include/linux/lustre_idl.h b/lustre/include/linux/lustre_idl.h index cadfb03..71f19fe 100644 --- a/lustre/include/linux/lustre_idl.h +++ b/lustre/include/linux/lustre_idl.h @@ -990,7 +990,9 @@ typedef enum { #define MTI_NAME_MAXLEN 64 #define MTI_UUID_MAXLEN MTI_NAME_MAXLEN + 5 -#define MTI_NIDS_MAX 10 /* match lustre_disk.h */ +/* each host can have multiple nids, and multiple failover hosts, and I don't + want to run out of room... */ +#define MTI_NIDS_MAX 64 /* match lustre_disk.h */ struct mgs_target_info { char mti_fsname[MTI_NAME_MAXLEN]; diff --git a/lustre/include/linux/obd_class.h b/lustre/include/linux/obd_class.h index eed0753..cebe4e4 100644 --- a/lustre/include/linux/obd_class.h +++ b/lustre/include/linux/obd_class.h @@ -1069,9 +1069,14 @@ static inline int obd_notify(struct obd_device *obd, void *data) { OBD_CHECK_DEV(obd); - if (!obd->obd_set_up) { - CERROR("obd %s not set up\n", obd->obd_name); - return -EINVAL; + + /* the check for async_recov is a complete hack - I'm hereby + overloading the meaning to also mean "this was called from + mds_postsetup". I know that my mds is able to handle notifies + by this point, and it needs to get them to execute mds_postrecov. */ + if (!obd->obd_set_up && !obd->obd_async_recov) { + CERROR("obd %s not set up, notifying anyhow\n", obd->obd_name); + return -EAGAIN; } if (!OBP(obd, notify)) { diff --git a/lustre/mds/handler.c b/lustre/mds/handler.c index 91a5f88..09386fd 100644 --- a/lustre/mds/handler.c +++ b/lustre/mds/handler.c @@ -2115,10 +2115,11 @@ int mds_postrecov(struct obd_device *obd) /* FIXME Does target_finish_recovery really need this to block? */ /* Notify the LOV, which will in turn call mds_notify for each tgt */ + /* This means that we have to fool obd_notify to think we're obd_set_up + during mds_lov_connect. */ obd_notify(obd->u.mds.mds_osc_obd, NULL, obd->obd_async_recov ? OBD_NOTIFY_SYNC_NONBLOCK : OBD_NOTIFY_SYNC, NULL); - //mds_lov_start_synchronize(obd, NULL, NULL, obd->obd_async_recov); /* quota recovery */ lquota_recovery(quota_interface, obd); diff --git a/lustre/mds/mds_lov.c b/lustre/mds/mds_lov.c index 4c0c4a0..7726861 100644 --- a/lustre/mds/mds_lov.c +++ b/lustre/mds/mds_lov.c @@ -368,7 +368,7 @@ int mds_lov_connect(struct obd_device *obd, char * lov_name) * set_nextid(). The class driver can help us here, because * it can use the obd_recovering flag to determine when the * the OBD is full available. */ - if (!obd->obd_recovering) + if (!obd->obd_recovering) rc = mds_postrecov(obd); RETURN(rc); @@ -749,6 +749,7 @@ int mds_notify(struct obd_device *obd, struct obd_device *watched, ENTRY; switch (ev) { + /* We only handle these: */ case OBD_NOTIFY_ACTIVE: case OBD_NOTIFY_SYNC: case OBD_NOTIFY_SYNC_NONBLOCK: @@ -759,7 +760,7 @@ int mds_notify(struct obd_device *obd, struct obd_device *watched, CDEBUG(D_WARNING, "notify %s ev=%d\n", watched->obd_name, ev); - if (strcmp(watched->obd_type->typ_name, LUSTRE_OSC_NAME)) { + if (strcmp(watched->obd_type->typ_name, LUSTRE_OSC_NAME) != 0) { CERROR("unexpected notification of %s %s!\n", watched->obd_type->typ_name, watched->obd_name); RETURN(-EINVAL); diff --git a/lustre/utils/mkfs_lustre.c b/lustre/utils/mkfs_lustre.c index 1fff034..54db5fb 100644 --- a/lustre/utils/mkfs_lustre.c +++ b/lustre/utils/mkfs_lustre.c @@ -189,31 +189,6 @@ static void run_command_out() } } -#if 0 -static int lnet_setup = 0; -static int lnet_start() -{ - ptl_initialize(0, NULL); - if (access("/proc/sys/lnet", X_OK) != 0) { - fprintf(stderr, "%s: The LNET module must be loaded to " - "determine local NIDs\n", progname); - return 1; - } - if (jt_ptl_get_nids(NULL) == -ENETDOWN) { - char *cmd[]={"network", "up"}; - jt_ptl_network(2, cmd); - lnet_setup++; - } - return 0; -} - -static void lnet_stop() -{ - char *cmd[]={"network", "down"}; - if (--lnet_setup == 0) - jt_ptl_network(2, cmd); -} -#endif /*============ disk dev functions ===================*/ @@ -910,7 +885,7 @@ int parse_opts(int argc, char *const argv[], struct mkfs_opts *mop, char *s1 = optarg, *s2; if (IS_MGS(&mop->mo_ldd)) { badopt(long_opt[longidx].name, - "non-MGMT MDT,OST"); + "non-MGS MDT,OST"); return 1; } while ((s2 = strsep(&s1, ","))) { @@ -1081,46 +1056,6 @@ int main(int argc, char *const argv[]) mop.mo_ldd.ldd_flags |= LDD_F_SV_TYPE_MGS; } -#if 0 - if (IS_MGS(&mop.mo_ldd) && (mop.mo_ldd.ldd_mgsnid_count == 0)) { - int i; - __u64 *nids; - - vprint("No mgs nids specified, using all local nids\n"); - ret = lnet_start(); - if (ret) - goto out; - i = jt_ptl_get_nids(&nids); - if (i < 0) { - fprintf(stderr, "%s: Can't find local nids " - "(is the lnet module loaded?)\n", progname); - } else { - if (i > 0) { - if (i > MTI_NIDS_MAX) - i = MTI_NIDS_MAX; - vprint("Adding %d local nids for MGS\n", i); - memcpy(mop.mo_ldd.ldd_mgsnid, nids, - sizeof(mop.mo_ldd.ldd_mgsnid)); - free(nids); - } - mop.mo_ldd.ldd_mgsnid_count = i; - } - } - - if (IS_MGS(&mop.mo_ldd) && mop.mo_ldd.ldd_failnid_count) { - /* Add failover nids to mgsnids if we start an MGS - (MDT must have all possible MGS nids for failover.) */ - int i = 0, j = mop.mo_ldd.ldd_mgsnid_count; - while (i < mop.mo_ldd.ldd_failnid_count) { - if (j >= MTI_NIDS_MAX) - break; - mop.mo_ldd.ldd_mgsnid[j++] = - mop.mo_ldd.ldd_failnid[i++]; - } - mop.mo_ldd.ldd_mgsnid_count = j; - } -#endif - if (!IS_MGS(&mop.mo_ldd) && (mop.mo_ldd.ldd_mgsnid_count == 0)) { fatal(); fprintf(stderr, "Must specify either --mgs or --mgsnid\n"); -- 1.8.3.1