From 18f19045904ae09104a3f9bcbe9e99f345e421b4 Mon Sep 17 00:00:00 2001 From: nathan Date: Mon, 19 Mar 2007 19:09:48 +0000 Subject: [PATCH] b=11778 i=adilger wait until config file is finished before allowing MDT clients to connect --- lustre/include/obd.h | 7 ++++-- lustre/include/obd_class.h | 4 +-- lustre/ldlm/ldlm_lib.c | 16 +++++++++--- lustre/mds/handler.c | 61 +++++++++++---------------------------------- lustre/mds/mds_lov.c | 6 +++++ lustre/obdclass/obd_mount.c | 3 +++ lustre/tests/conf-sanity.sh | 7 ++++-- 7 files changed, 47 insertions(+), 57 deletions(-) diff --git a/lustre/include/obd.h b/lustre/include/obd.h index 4f3bd36..b795690 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -691,7 +691,9 @@ enum obd_notify_event { OBD_NOTIFY_OCD, /* Sync request */ OBD_NOTIFY_SYNC_NONBLOCK, - OBD_NOTIFY_SYNC + OBD_NOTIFY_SYNC, + /* Configuration event */ + OBD_NOTIFY_CONFIG }; /* @@ -728,7 +730,8 @@ struct obd_device { obd_starting:1, /* started setup */ obd_force:1, /* cleanup with > 0 obd refcount */ obd_fail:1, /* cleanup with failover */ - obd_async_recov:1; /* allow asyncronous orphan cleanup */ + obd_async_recov:1, /* allow asyncronous orphan cleanup */ + obd_no_conn:1; /* deny new connections */ atomic_t obd_refcount; /* uuid-export hash body */ struct lustre_class_hash_body *obd_uuid_hash_body; diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h index 9802e16..dcd0876 100644 --- a/lustre/include/obd_class.h +++ b/lustre/include/obd_class.h @@ -1276,10 +1276,8 @@ static inline int obd_notify(struct obd_device *obd, RETURN(-EINVAL); } - if (!OBP(obd, notify)) { - CERROR("obd %s has no notify handler\n", obd->obd_name); + if (!OBP(obd, notify)) RETURN(-ENOSYS); - } OBD_COUNTER_INCREMENT(obd, notify); RETURN(OBP(obd, notify)(obd, watched, ev, data)); diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index cbb963d..3122ebb 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -569,13 +569,21 @@ int target_handle_connect(struct ptlrpc_request *req, svc_handler_t handler) /* end COMPAT_146 */ if (!target || target->obd_stopping || !target->obd_set_up) { - DEBUG_REQ(D_ERROR, req, "UUID '%s' is not available " - " for connect (%s)", str, - !target ? "no target" : - (target->obd_stopping ? "stopping" : "not set up")); + LCONSOLE_ERROR("UUID '%s' is not available " + " for connect (%s)", str, + !target ? "no target" : + (target->obd_stopping ? "stopping" : + "not set up")); GOTO(out, rc = -ENODEV); } + if (target->obd_no_conn) { + LCONSOLE_WARN("%s: temporarily refusing client connection " + "from %s\n", target->obd_name, + libcfs_nid2str(req->rq_peer.nid)); + GOTO(out, rc = -EAGAIN); + } + /* Make sure the target isn't cleaned up while we're here. Yes, there's still a race between the above check and our incref here. Really, class_uuid2obd should take the ref. */ diff --git a/lustre/mds/handler.c b/lustre/mds/handler.c index 8cc1aba..04d0802 100644 --- a/lustre/mds/handler.c +++ b/lustre/mds/handler.c @@ -1877,13 +1877,13 @@ static int mds_setup(struct obd_device *obd, obd_count len, void *buf) struct lprocfs_static_vars lvars; struct lustre_cfg* lcfg = buf; struct mds_obd *mds = &obd->u.mds; + struct lustre_sb_info *lsi; struct lustre_mount_info *lmi; struct vfsmount *mnt; struct obd_uuid uuid; __u8 *uuid_ptr; - char *options, *str, *label; + char *str, *label; char ns_name[48]; - unsigned long page; int rc = 0; ENTRY; @@ -1893,55 +1893,24 @@ static int mds_setup(struct obd_device *obd, obd_count len, void *buf) offsetof(struct obd_device, u.mds.mds_obt)); if (lcfg->lcfg_bufcount < 3) - RETURN(rc = -EINVAL); + RETURN(-EINVAL); if (LUSTRE_CFG_BUFLEN(lcfg, 1) == 0 || LUSTRE_CFG_BUFLEN(lcfg, 2) == 0) - RETURN(rc = -EINVAL); + RETURN(-EINVAL); lmi = server_get_mount(obd->obd_name); - if (lmi) { - /* We already mounted in lustre_fill_super. - lcfg bufs 1, 2, 4 (device, fstype, mount opts) are ignored.*/ - struct lustre_sb_info *lsi = s2lsi(lmi->lmi_sb); - fsoptions_to_mds_flags(mds, lsi->lsi_ldd->ldd_mount_opts); - fsoptions_to_mds_flags(mds, lsi->lsi_lmd->lmd_opts); - mnt = lmi->lmi_mnt; - obd->obd_fsops = fsfilt_get_ops(MT_STR(lsi->lsi_ldd)); - } else { - /* old path - used by lctl */ - CERROR("Using old MDS mount method\n"); - page = __get_free_page(GFP_KERNEL); - if (!page) - RETURN(-ENOMEM); - - options = (char *)page; - memset(options, 0, CFS_PAGE_SIZE); - - /* here we use "iopen_nopriv" hardcoded, because it affects - * MDS utility and the rest of options are passed by mount - * options. Probably this should be moved to somewhere else - * like startup scripts or lconf. */ - strcpy(options, "iopen_nopriv"); - - if (LUSTRE_CFG_BUFLEN(lcfg, 4) > 0 && lustre_cfg_buf(lcfg, 4)) { - sprintf(options + strlen(options), ",%s", - lustre_cfg_string(lcfg, 4)); - fsoptions_to_mds_flags(mds, options); - } - - mnt = do_kern_mount(lustre_cfg_string(lcfg, 2), 0, - lustre_cfg_string(lcfg, 1), - (void *)options); - free_page(page); - if (IS_ERR(mnt)) { - rc = PTR_ERR(mnt); - LCONSOLE_ERROR("Can't mount disk %s (%d)\n", - lustre_cfg_string(lcfg, 1), rc); - RETURN(rc); - } - - obd->obd_fsops = fsfilt_get_ops(lustre_cfg_string(lcfg, 2)); + if (!lmi) { + CERROR("Not mounted in lustre_fill_super?\n"); + RETURN(-EINVAL); } + + /* We mounted in lustre_fill_super. + lcfg bufs 1, 2, 4 (device, fstype, mount opts) are ignored.*/ + lsi = s2lsi(lmi->lmi_sb); + fsoptions_to_mds_flags(mds, lsi->lsi_ldd->ldd_mount_opts); + fsoptions_to_mds_flags(mds, lsi->lsi_lmd->lmd_opts); + mnt = lmi->lmi_mnt; + obd->obd_fsops = fsfilt_get_ops(MT_STR(lsi->lsi_ldd)); if (IS_ERR(obd->obd_fsops)) GOTO(err_put, rc = PTR_ERR(obd->obd_fsops)); diff --git a/lustre/mds/mds_lov.c b/lustre/mds/mds_lov.c index 43278a2..a411b89 100644 --- a/lustre/mds/mds_lov.c +++ b/lustre/mds/mds_lov.c @@ -347,6 +347,9 @@ int mds_lov_connect(struct obd_device *obd, char * lov_name) lov_name, rc); GOTO(err_discon, rc); } + + /* Deny new client connections until we are sure we have some OSTs */ + obd->obd_no_conn = 1; rc = mds_lov_read_objids(obd); if (rc) { @@ -766,6 +769,9 @@ int mds_notify(struct obd_device *obd, struct obd_device *watched, case OBD_NOTIFY_SYNC: case OBD_NOTIFY_SYNC_NONBLOCK: break; + case OBD_NOTIFY_CONFIG: + /* Open for clients */ + obd->obd_no_conn = 0; default: RETURN(0); } diff --git a/lustre/obdclass/obd_mount.c b/lustre/obdclass/obd_mount.c index 8822965..33d2d5d 100644 --- a/lustre/obdclass/obd_mount.c +++ b/lustre/obdclass/obd_mount.c @@ -1092,6 +1092,9 @@ out_mgc: obd_iocontrol(OBD_IOC_ABORT_RECOVERY, obd->obd_self_export, 0, NULL, NULL); } + + /* log has been fully processed */ + obd_notify(obd, NULL, OBD_NOTIFY_CONFIG, 0); } RETURN(rc); diff --git a/lustre/tests/conf-sanity.sh b/lustre/tests/conf-sanity.sh index 46bad10..757d9cc 100644 --- a/lustre/tests/conf-sanity.sh +++ b/lustre/tests/conf-sanity.sh @@ -1057,7 +1057,7 @@ test_32a() { $TUNEFS $TMP/$tdir/mds || error "tunefs failed" # nids are wrong, so client wont work, but server should start - start mds $TMP/$tdir/mds "-o loop" || return 3 + start mds $TMP/$tdir/mds "-o loop,exclude=lustre-OST0000" || return 3 local UUID=$(cat $LPROC/mds/lustre-MDT0000/uuid) echo MDS uuid $UUID [ "$UUID" == "mdsA_UUID" ] || error "UUID is wrong: $UUID" @@ -1081,7 +1081,10 @@ test_32a() { # With a new good MDT failover nid, we should be able to mount a client # (but it cant talk to OST) + local OLDMOUNTOPT=$MOUNTOPT + MOUNTOPT="exclude=lustre-OST0000" mount_client $MOUNT + MOUNTOPT=$OLDMOUNTOPT set_and_check "cat $LPROC/mdc/*/max_rpcs_in_flight" "lustre-MDT0000.mdc.max_rpcs_in_flight" || return 11 zconf_umount `hostname` $MOUNT -f @@ -1089,7 +1092,7 @@ test_32a() { # mount a second time to make sure we didnt leave upgrade flag on $TUNEFS --dryrun $TMP/$tdir/mds || error "tunefs failed" - start mds $TMP/$tdir/mds "-o loop" || return 12 + start mds $TMP/$tdir/mds "-o loop,exclude=lustre-OST0000" || return 12 cleanup_nocli [ -d $TMP/$tdir ] && rm -rf $TMP/$tdir -- 1.8.3.1