From 87319353501f4520284186e9e8bdb57695f90feb Mon Sep 17 00:00:00 2001 From: pravins Date: Tue, 8 Jul 2008 17:32:01 +0000 Subject: [PATCH] b=15516 i=tom.wang i=yury.umanets This patch is first step towards shifting mds-lov work to new lustre mds stack. - kill OBD_CLEANUP_SELF_EXP - added rw_sem to serialize mds_notify and mds_precleanup - kill OBD_CLEANUP_OBD --- lustre/include/obd.h | 10 ++-------- lustre/lmv/lmv_obd.c | 2 +- lustre/lov/lov_obd.c | 4 ---- lustre/mdc/mdc_request.c | 4 ---- lustre/mds/handler.c | 17 +++++++---------- lustre/mds/mds_lov.c | 16 +++++----------- lustre/mgc/libmgc.c | 4 ---- lustre/mgc/mgc_request.c | 4 ---- lustre/mgs/mgs_handler.c | 4 ---- lustre/obdclass/obd_config.c | 17 ++++------------- lustre/obdfilter/filter.c | 4 ---- lustre/osc/osc_request.c | 6 +----- 12 files changed, 20 insertions(+), 72 deletions(-) diff --git a/lustre/include/obd.h b/lustre/include/obd.h index e5804fe..553f2ca 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -556,6 +556,7 @@ struct mds_obd { /* for capability keys update */ struct lustre_capa_key *mds_capa_keys; + struct rw_semaphore mds_notify_lock; }; /* lov objid */ @@ -997,15 +998,8 @@ struct obd_device { enum obd_cleanup_stage { /* Special case hack for MDS LOVs */ OBD_CLEANUP_EARLY, -/* Precleanup stage 1, we must make sure all exports (other than the - self-export) get destroyed. */ +/* can be directly mapped to .ldto_device_fini() */ OBD_CLEANUP_EXPORTS, -/* Precleanup stage 2, do other type-specific cleanup requiring the - self-export. */ - OBD_CLEANUP_SELF_EXP, -/* FIXME we should eliminate the "precleanup" function and make them stages - of the "cleanup" function. */ - OBD_CLEANUP_OBD, }; /* get/set_info keys */ diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c index 73aba2a..82c33cd 100644 --- a/lustre/lmv/lmv_obd.c +++ b/lustre/lmv/lmv_obd.c @@ -2532,7 +2532,7 @@ static int lmv_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage) /* XXX: here should be calling obd_precleanup() down to * stack. */ break; - case OBD_CLEANUP_SELF_EXP: + case OBD_CLEANUP_EXPORTS: rc = obd_llog_finish(obd, 0); if (rc != 0) CERROR("failed to cleanup llogging subsystems\n"); diff --git a/lustre/lov/lov_obd.c b/lustre/lov/lov_obd.c index 43b18e78..b4b4263 100644 --- a/lustre/lov/lov_obd.c +++ b/lustre/lov/lov_obd.c @@ -906,14 +906,10 @@ static int lov_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage) break; } case OBD_CLEANUP_EXPORTS: - break; - case OBD_CLEANUP_SELF_EXP: rc = obd_llog_finish(obd, 0); if (rc != 0) CERROR("failed to cleanup llogging subsystems\n"); break; - case OBD_CLEANUP_OBD: - break; } RETURN(rc); } diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c index 2e3e349..f17e571 100644 --- a/lustre/mdc/mdc_request.c +++ b/lustre/mdc/mdc_request.c @@ -1603,10 +1603,6 @@ static int mdc_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage) if (rc != 0) CERROR("failed to cleanup llogging subsystems\n"); break; - case OBD_CLEANUP_SELF_EXP: - break; - case OBD_CLEANUP_OBD: - break; } RETURN(rc); } diff --git a/lustre/mds/handler.c b/lustre/mds/handler.c index 9621687..60bb4cb 100644 --- a/lustre/mds/handler.c +++ b/lustre/mds/handler.c @@ -166,7 +166,6 @@ static int mds_lov_clean(struct obd_device *obd) /* Cleanup the lov */ obd_disconnect(mds->mds_osc_exp); class_manual_cleanup(osc); - mds->mds_osc_exp = NULL; RETURN(0); } @@ -261,27 +260,22 @@ static int mds_lov_early_clean(struct obd_device *obd) static int mds_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage) { int rc = 0; + struct mds_obd *mds = &obd->u.mds; ENTRY; switch (stage) { case OBD_CLEANUP_EARLY: break; case OBD_CLEANUP_EXPORTS: - /*XXX Use this for mdd mds cleanup, so comment out - *this target_cleanup_recovery for this tmp MDD MDS - *Wangdi*/ - if (strncmp(obd->obd_name, MDD_OBD_NAME, strlen(MDD_OBD_NAME))) - target_cleanup_recovery(obd); mds_lov_early_clean(obd); - break; - case OBD_CLEANUP_SELF_EXP: + down_write(&mds->mds_notify_lock); mds_lov_disconnect(obd); mds_lov_clean(obd); llog_cleanup(llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT)); llog_cleanup(llog_get_context(obd, LLOG_LOVEA_ORIG_CTXT)); rc = obd_llog_finish(obd, 0); - break; - case OBD_CLEANUP_OBD: + mds->mds_osc_exp = NULL; + up_write(&mds->mds_notify_lock); break; } RETURN(rc); @@ -356,6 +350,7 @@ static int mds_cmd_setup(struct obd_device *obd, struct lustre_cfg *lcfg) */ atomic_dec(&lsi->lsi_mounts); mntput(mnt); + init_rwsem(&mds->mds_notify_lock); obd->obd_fsops = fsfilt_get_ops(MT_STR(lsi->lsi_ldd)); mds_init_ctxt(obd, mnt); @@ -426,6 +421,8 @@ static int mds_cmd_cleanup(struct obd_device *obd) int rc = 0; ENTRY; + mds->mds_osc_exp = NULL; + if (obd->obd_fail) LCONSOLE_WARN("%s: shutting down for failover; client state " "will be preserved.\n", obd->obd_name); diff --git a/lustre/mds/mds_lov.c b/lustre/mds/mds_lov.c index 506b648..9a090cf 100644 --- a/lustre/mds/mds_lov.c +++ b/lustre/mds/mds_lov.c @@ -646,6 +646,9 @@ static int __mds_lov_synchronize(void *data) ENTRY; OBD_FREE(mlsi, sizeof(*mlsi)); + down_read(&mds->mds_notify_lock); + if (obd->obd_stopping || obd->obd_fail) + GOTO(out, rc = -ENODEV); LASSERT(obd); LASSERT(watched); @@ -653,13 +656,11 @@ static int __mds_lov_synchronize(void *data) LASSERT(uuid); OBD_RACE(OBD_FAIL_MDS_LOV_SYNC_RACE); - rc = mds_lov_update_mds(obd, watched, idx); if (rc != 0) { CERROR("%s failed at update_mds: %d\n", obd_uuid2str(uuid), rc); GOTO(out, rc); } - mgi.group = FILTER_GROUP_MDS0 + mds->mds_id; mgi.uuid = uuid; @@ -674,10 +675,9 @@ static int __mds_lov_synchronize(void *data) ctxt = llog_get_context(obd, LLOG_MDS_OST_ORIG_CTXT); if (!ctxt) - RETURN(-ENODEV); + GOTO(out, rc = -ENODEV); OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_LLOG_SYNC_TIMEOUT, 60); - rc = llog_connect(ctxt, obd->u.mds.mds_lov_desc.ld_tgt_count, NULL, NULL, uuid); llog_ctxt_put(ctxt); @@ -689,13 +689,6 @@ static int __mds_lov_synchronize(void *data) LCONSOLE_INFO("MDS %s: %s now active, resetting orphans\n", obd->obd_name, obd_uuid2str(uuid)); - /* - * FIXME: this obd_stopping was useless, - * since obd in mdt layer was set - */ - if (obd->obd_stopping) - GOTO(out, rc = -ENODEV); - rc = mds_lov_clear_orphans(mds, uuid); if (rc != 0) { CERROR("%s failed at mds_lov_clear_orphans: %d\n", @@ -714,6 +707,7 @@ static int __mds_lov_synchronize(void *data) } EXIT; out: + up_read(&mds->mds_notify_lock); if (rc) { /* Deactivate it for safety */ CERROR("%s sync failed %d, deactivating\n", obd_uuid2str(uuid), diff --git a/lustre/mgc/libmgc.c b/lustre/mgc/libmgc.c index da6975f..3eae830 100644 --- a/lustre/mgc/libmgc.c +++ b/lustre/mgc/libmgc.c @@ -74,14 +74,10 @@ static int mgc_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage) switch (stage) { case OBD_CLEANUP_EARLY: case OBD_CLEANUP_EXPORTS: - break; - case OBD_CLEANUP_SELF_EXP: rc = obd_llog_finish(obd, 0); if (rc != 0) CERROR("failed to cleanup llogging subsystems\n"); break; - case OBD_CLEANUP_OBD: - break; } RETURN(rc); } diff --git a/lustre/mgc/mgc_request.c b/lustre/mgc/mgc_request.c index e7e05b8..daee68a 100644 --- a/lustre/mgc/mgc_request.c +++ b/lustre/mgc/mgc_request.c @@ -471,10 +471,6 @@ static int mgc_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage) if (rc != 0) CERROR("failed to cleanup llogging subsystems\n"); break; - case OBD_CLEANUP_SELF_EXP: - break; - case OBD_CLEANUP_OBD: - break; } RETURN(rc); } diff --git a/lustre/mgs/mgs_handler.c b/lustre/mgs/mgs_handler.c index 79d6f0f..b0ecdcd 100644 --- a/lustre/mgs/mgs_handler.c +++ b/lustre/mgs/mgs_handler.c @@ -258,12 +258,8 @@ static int mgs_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage) switch (stage) { case OBD_CLEANUP_EARLY: case OBD_CLEANUP_EXPORTS: - break; - case OBD_CLEANUP_SELF_EXP: rc = obd_llog_finish(obd, 0); break; - case OBD_CLEANUP_OBD: - break; } RETURN(rc); } diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c index f0bdd83..25f9d5a 100644 --- a/lustre/obdclass/obd_config.c +++ b/lustre/obdclass/obd_config.c @@ -455,8 +455,7 @@ int class_cleanup(struct obd_device *obd, struct lustre_cfg *lcfg) /* destroy a nid-stats hash body */ lustre_hash_exit(&obd->obd_nid_stats_hash_body); - /* Precleanup stage 1, we must make sure all exports (other than the - self-export) get destroyed. */ + /* Precleanup, we must make sure all exports get destroyed. */ err = obd_precleanup(obd, OBD_CLEANUP_EXPORTS); if (err) CERROR("Precleanup %s returned %d\n", @@ -489,17 +488,9 @@ void class_decref(struct obd_device *obd) CDEBUG(D_INFO, "Decref %s (%p) now %d\n", obd->obd_name, obd, refs); if ((refs == 1) && obd->obd_stopping) { - /* All exports (other than the self-export) have been - destroyed; there should be no more in-progress ops - by this point.*/ - /* if we're not stopping, we didn't finish setup */ - /* Precleanup stage 2, do other type-specific - cleanup requiring the self-export. */ - err = obd_precleanup(obd, OBD_CLEANUP_SELF_EXP); - if (err) - CERROR("Precleanup %s returned %d\n", - obd->obd_name, err); - + /* All exports have been destroyed; there should + be no more in-progress ops by this point.*/ + spin_lock(&obd->obd_self_export->exp_lock); obd->obd_self_export->exp_flags |= (obd->obd_fail ? OBD_OPT_FAILOVER : 0) | diff --git a/lustre/obdfilter/filter.c b/lustre/obdfilter/filter.c index f740f46..8f9ac20 100644 --- a/lustre/obdfilter/filter.c +++ b/lustre/obdfilter/filter.c @@ -2455,10 +2455,6 @@ static int filter_precleanup(struct obd_device *obd, target_cleanup_recovery(obd); rc = filter_llog_preclean(obd); break; - case OBD_CLEANUP_SELF_EXP: - break; - case OBD_CLEANUP_OBD: - break; } RETURN(rc); } diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index 17c7e1d..24f7530 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -3989,11 +3989,7 @@ static int osc_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage) if (rc != 0) CERROR("failed to cleanup llogging subsystems\n"); break; - } - case OBD_CLEANUP_SELF_EXP: - break; - case OBD_CLEANUP_OBD: - break; + } } RETURN(rc); } -- 1.8.3.1