b=15516

i=tom.wang
i=yury.umanets

This patch is first step towards shifting mds-lov work to new
lustre mds stack.
        -       kill OBD_CLEANUP_SELF_EXP
        -       added rw_sem to serialize mds_notify and mds_precleanup
        -       kill OBD_CLEANUP_OBD

diff --git a/lustre/include/obd.h b/lustre/include/obd.h
index e5804fe..553f2ca 100644 (file)
--- a/lustre/include/obd.h
+++ b/lustre/include/obd.h
@@ -556,6 +556,7 @@ struct mds_obd {
 
         /* for capability keys update */
         struct lustre_capa_key          *mds_capa_keys;
 
         /* for capability keys update */
         struct lustre_capa_key          *mds_capa_keys;

+        struct rw_semaphore             mds_notify_lock;

 };
 
 /* lov objid */
 };
 
 /* lov objid */


@@ -997,15 +998,8 @@ struct obd_device {
 enum obd_cleanup_stage {
 /* Special case hack for MDS LOVs */
         OBD_CLEANUP_EARLY,
 enum obd_cleanup_stage {
 /* Special case hack for MDS LOVs */
         OBD_CLEANUP_EARLY,

-/* Precleanup stage 1, we must make sure all exports (other than the
-   self-export) get destroyed. */
+/* can be directly mapped to .ldto_device_fini() */

         OBD_CLEANUP_EXPORTS,
         OBD_CLEANUP_EXPORTS,

-/* Precleanup stage 2,  do other type-specific cleanup requiring the
-   self-export. */
-        OBD_CLEANUP_SELF_EXP,
-/* FIXME we should eliminate the "precleanup" function and make them stages
-   of the "cleanup" function. */
-        OBD_CLEANUP_OBD,

 };
 
 /* get/set_info keys */
 };
 
 /* get/set_info keys */

diff --git a/lustre/lmv/lmv_obd.c b/lustre/lmv/lmv_obd.c
index 73aba2a..82c33cd 100644 (file)
--- a/lustre/lmv/lmv_obd.c
+++ b/lustre/lmv/lmv_obd.c
@@ -2532,7 +2532,7 @@ static int lmv_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage)
                 /* XXX: here should be calling obd_precleanup() down to
                  * stack. */
                 break;
                 /* XXX: here should be calling obd_precleanup() down to
                  * stack. */
                 break;

-        case OBD_CLEANUP_SELF_EXP:
+        case OBD_CLEANUP_EXPORTS:

                 rc = obd_llog_finish(obd, 0);
                 if (rc != 0)
                         CERROR("failed to cleanup llogging subsystems\n");
                 rc = obd_llog_finish(obd, 0);
                 if (rc != 0)
                         CERROR("failed to cleanup llogging subsystems\n");

diff --git a/lustre/lov/lov_obd.c b/lustre/lov/lov_obd.c
index 43b18e7..b4b4263 100644 (file)
--- a/lustre/lov/lov_obd.c
+++ b/lustre/lov/lov_obd.c
@@ -906,14 +906,10 @@ static int lov_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage)
                 break;
         }
         case OBD_CLEANUP_EXPORTS:
                 break;
         }
         case OBD_CLEANUP_EXPORTS:

-                break;
-        case OBD_CLEANUP_SELF_EXP:

                 rc = obd_llog_finish(obd, 0);
                 if (rc != 0)
                         CERROR("failed to cleanup llogging subsystems\n");
                 break;
                 rc = obd_llog_finish(obd, 0);
                 if (rc != 0)
                         CERROR("failed to cleanup llogging subsystems\n");
                 break;

-        case OBD_CLEANUP_OBD:
-                break;

         }
         RETURN(rc);
 }
         }
         RETURN(rc);
 }

diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c
index 2e3e349..f17e571 100644 (file)
--- a/lustre/mdc/mdc_request.c
+++ b/lustre/mdc/mdc_request.c
@@ -1603,10 +1603,6 @@ static int mdc_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage)
                 if (rc != 0)
                         CERROR("failed to cleanup llogging subsystems\n");
                 break;
                 if (rc != 0)
                         CERROR("failed to cleanup llogging subsystems\n");
                 break;

-        case OBD_CLEANUP_SELF_EXP:
-                break;
-        case OBD_CLEANUP_OBD:
-                break;

         }
         RETURN(rc);
 }
         }
         RETURN(rc);
 }

diff --git a/lustre/mds/handler.c b/lustre/mds/handler.c
index 9621687..60bb4cb 100644 (file)
--- a/lustre/mds/handler.c
+++ b/lustre/mds/handler.c
@@ -166,7 +166,6 @@ static int mds_lov_clean(struct obd_device *obd)
         /* Cleanup the lov */
         obd_disconnect(mds->mds_osc_exp);
         class_manual_cleanup(osc);
         /* Cleanup the lov */
         obd_disconnect(mds->mds_osc_exp);
         class_manual_cleanup(osc);

-        mds->mds_osc_exp = NULL;

 
         RETURN(0);
 }
 
         RETURN(0);
 }


@@ -261,27 +260,22 @@ static int mds_lov_early_clean(struct obd_device *obd)
 static int mds_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage)
 {
         int rc = 0;
 static int mds_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage)
 {
         int rc = 0;

+        struct mds_obd *mds = &obd->u.mds;

         ENTRY;
 
         switch (stage) {
         case OBD_CLEANUP_EARLY:
                 break;
         case OBD_CLEANUP_EXPORTS:
         ENTRY;
 
         switch (stage) {
         case OBD_CLEANUP_EARLY:
                 break;
         case OBD_CLEANUP_EXPORTS:

-                /*XXX Use this for mdd mds cleanup, so comment out
-                 *this target_cleanup_recovery for this tmp MDD MDS
-                 *Wangdi*/
-                if (strncmp(obd->obd_name, MDD_OBD_NAME, strlen(MDD_OBD_NAME)))
-                        target_cleanup_recovery(obd);

                 mds_lov_early_clean(obd);
                 mds_lov_early_clean(obd);

-                break;
-        case OBD_CLEANUP_SELF_EXP:
+                down_write(&mds->mds_notify_lock);

                 mds_lov_disconnect(obd);
                 mds_lov_clean(obd);
                 llog_cleanup(llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT));
                 llog_cleanup(llog_get_context(obd, LLOG_LOVEA_ORIG_CTXT));
                 rc = obd_llog_finish(obd, 0);
                 mds_lov_disconnect(obd);
                 mds_lov_clean(obd);
                 llog_cleanup(llog_get_context(obd, LLOG_CONFIG_ORIG_CTXT));
                 llog_cleanup(llog_get_context(obd, LLOG_LOVEA_ORIG_CTXT));
                 rc = obd_llog_finish(obd, 0);

-                break;
-        case OBD_CLEANUP_OBD:
+                mds->mds_osc_exp = NULL;
+                up_write(&mds->mds_notify_lock);

                 break;
         }
         RETURN(rc);
                 break;
         }
         RETURN(rc);


@@ -356,6 +350,7 @@ static int mds_cmd_setup(struct obd_device *obd, struct lustre_cfg *lcfg)
          */
         atomic_dec(&lsi->lsi_mounts);
         mntput(mnt);
          */
         atomic_dec(&lsi->lsi_mounts);
         mntput(mnt);

+        init_rwsem(&mds->mds_notify_lock);

 
         obd->obd_fsops = fsfilt_get_ops(MT_STR(lsi->lsi_ldd));
         mds_init_ctxt(obd, mnt);
 
         obd->obd_fsops = fsfilt_get_ops(MT_STR(lsi->lsi_ldd));
         mds_init_ctxt(obd, mnt);


@@ -426,6 +421,8 @@ static int mds_cmd_cleanup(struct obd_device *obd)
         int rc = 0;
         ENTRY;
 
         int rc = 0;
         ENTRY;
 

+        mds->mds_osc_exp = NULL;
+

         if (obd->obd_fail)
                 LCONSOLE_WARN("%s: shutting down for failover; client state "
                               "will be preserved.\n", obd->obd_name);
         if (obd->obd_fail)
                 LCONSOLE_WARN("%s: shutting down for failover; client state "
                               "will be preserved.\n", obd->obd_name);

diff --git a/lustre/mds/mds_lov.c b/lustre/mds/mds_lov.c
index 506b648..9a090cf 100644 (file)
--- a/lustre/mds/mds_lov.c
+++ b/lustre/mds/mds_lov.c
@@ -646,6 +646,9 @@ static int __mds_lov_synchronize(void *data)
         ENTRY;
 
         OBD_FREE(mlsi, sizeof(*mlsi));
         ENTRY;
 
         OBD_FREE(mlsi, sizeof(*mlsi));

+        down_read(&mds->mds_notify_lock);
+        if (obd->obd_stopping || obd->obd_fail)
+                GOTO(out, rc = -ENODEV);

 
         LASSERT(obd);
         LASSERT(watched);
 
         LASSERT(obd);
         LASSERT(watched);


@@ -653,13 +656,11 @@ static int __mds_lov_synchronize(void *data)
         LASSERT(uuid);
 
         OBD_RACE(OBD_FAIL_MDS_LOV_SYNC_RACE);
         LASSERT(uuid);
 
         OBD_RACE(OBD_FAIL_MDS_LOV_SYNC_RACE);

-

         rc = mds_lov_update_mds(obd, watched, idx);
         if (rc != 0) {
                 CERROR("%s failed at update_mds: %d\n", obd_uuid2str(uuid), rc);
                 GOTO(out, rc);
         }
         rc = mds_lov_update_mds(obd, watched, idx);
         if (rc != 0) {
                 CERROR("%s failed at update_mds: %d\n", obd_uuid2str(uuid), rc);
                 GOTO(out, rc);
         }

-

         mgi.group = FILTER_GROUP_MDS0 + mds->mds_id;
         mgi.uuid = uuid;
 
         mgi.group = FILTER_GROUP_MDS0 + mds->mds_id;
         mgi.uuid = uuid;
 


@@ -674,10 +675,9 @@ static int __mds_lov_synchronize(void *data)
 
         ctxt = llog_get_context(obd, LLOG_MDS_OST_ORIG_CTXT);
         if (!ctxt) 
 
         ctxt = llog_get_context(obd, LLOG_MDS_OST_ORIG_CTXT);
         if (!ctxt) 

-                RETURN(-ENODEV);
+                GOTO(out, rc = -ENODEV);

 
         OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_LLOG_SYNC_TIMEOUT, 60);
 
         OBD_FAIL_TIMEOUT(OBD_FAIL_MDS_LLOG_SYNC_TIMEOUT, 60);

-

         rc = llog_connect(ctxt, obd->u.mds.mds_lov_desc.ld_tgt_count, 
                           NULL, NULL, uuid); 
         llog_ctxt_put(ctxt);
         rc = llog_connect(ctxt, obd->u.mds.mds_lov_desc.ld_tgt_count, 
                           NULL, NULL, uuid); 
         llog_ctxt_put(ctxt);


@@ -689,13 +689,6 @@ static int __mds_lov_synchronize(void *data)
 
         LCONSOLE_INFO("MDS %s: %s now active, resetting orphans\n",
               obd->obd_name, obd_uuid2str(uuid));
 
         LCONSOLE_INFO("MDS %s: %s now active, resetting orphans\n",
               obd->obd_name, obd_uuid2str(uuid));

-        /*
-         * FIXME: this obd_stopping was useless, 
-         * since obd in mdt layer was set
-         */
-        if (obd->obd_stopping)
-                GOTO(out, rc = -ENODEV);
-

         rc = mds_lov_clear_orphans(mds, uuid);
         if (rc != 0) {
                 CERROR("%s failed at mds_lov_clear_orphans: %d\n",
         rc = mds_lov_clear_orphans(mds, uuid);
         if (rc != 0) {
                 CERROR("%s failed at mds_lov_clear_orphans: %d\n",


@@ -714,6 +707,7 @@ static int __mds_lov_synchronize(void *data)
         }
         EXIT;
 out:
         }
         EXIT;
 out:

+        up_read(&mds->mds_notify_lock);

         if (rc) {
                 /* Deactivate it for safety */
                 CERROR("%s sync failed %d, deactivating\n", obd_uuid2str(uuid),
         if (rc) {
                 /* Deactivate it for safety */
                 CERROR("%s sync failed %d, deactivating\n", obd_uuid2str(uuid),

diff --git a/lustre/mgc/libmgc.c b/lustre/mgc/libmgc.c
index da6975f..3eae830 100644 (file)
--- a/lustre/mgc/libmgc.c
+++ b/lustre/mgc/libmgc.c
@@ -74,14 +74,10 @@ static int mgc_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage)
         switch (stage) {
         case OBD_CLEANUP_EARLY: 
         case OBD_CLEANUP_EXPORTS:
         switch (stage) {
         case OBD_CLEANUP_EARLY: 
         case OBD_CLEANUP_EXPORTS:

-                break;
-        case OBD_CLEANUP_SELF_EXP:

                 rc = obd_llog_finish(obd, 0);
                 if (rc != 0)
                         CERROR("failed to cleanup llogging subsystems\n");
                 break;
                 rc = obd_llog_finish(obd, 0);
                 if (rc != 0)
                         CERROR("failed to cleanup llogging subsystems\n");
                 break;

-        case OBD_CLEANUP_OBD:
-                break;

         }
         RETURN(rc);
 }
         }
         RETURN(rc);
 }

diff --git a/lustre/mgc/mgc_request.c b/lustre/mgc/mgc_request.c
index e7e05b8..daee68a 100644 (file)
--- a/lustre/mgc/mgc_request.c
+++ b/lustre/mgc/mgc_request.c
@@ -471,10 +471,6 @@ static int mgc_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage)
                 if (rc != 0)
                         CERROR("failed to cleanup llogging subsystems\n");
                 break;
                 if (rc != 0)
                         CERROR("failed to cleanup llogging subsystems\n");
                 break;

-        case OBD_CLEANUP_SELF_EXP:
-                break;
-        case OBD_CLEANUP_OBD:
-                break;

         }
         RETURN(rc);
 }
         }
         RETURN(rc);
 }

diff --git a/lustre/mgs/mgs_handler.c b/lustre/mgs/mgs_handler.c
index 79d6f0f..b0ecdcd 100644 (file)
--- a/lustre/mgs/mgs_handler.c
+++ b/lustre/mgs/mgs_handler.c
@@ -258,12 +258,8 @@ static int mgs_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage)
         switch (stage) {
         case OBD_CLEANUP_EARLY:
         case OBD_CLEANUP_EXPORTS:
         switch (stage) {
         case OBD_CLEANUP_EARLY:
         case OBD_CLEANUP_EXPORTS:

-                break;
-        case OBD_CLEANUP_SELF_EXP:

                 rc = obd_llog_finish(obd, 0);
                 break;
                 rc = obd_llog_finish(obd, 0);
                 break;

-        case OBD_CLEANUP_OBD:
-                break;

         }
         RETURN(rc);
 }
         }
         RETURN(rc);
 }

diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c
index f0bdd83..25f9d5a 100644 (file)
--- a/lustre/obdclass/obd_config.c
+++ b/lustre/obdclass/obd_config.c
@@ -455,8 +455,7 @@ int class_cleanup(struct obd_device *obd, struct lustre_cfg *lcfg)
         /* destroy a nid-stats hash body */
         lustre_hash_exit(&obd->obd_nid_stats_hash_body);
 
         /* destroy a nid-stats hash body */
         lustre_hash_exit(&obd->obd_nid_stats_hash_body);
 

-        /* Precleanup stage 1, we must make sure all exports (other than the
-           self-export) get destroyed. */
+        /* Precleanup, we must make sure all exports get destroyed. */

         err = obd_precleanup(obd, OBD_CLEANUP_EXPORTS);
         if (err)
                 CERROR("Precleanup %s returned %d\n",
         err = obd_precleanup(obd, OBD_CLEANUP_EXPORTS);
         if (err)
                 CERROR("Precleanup %s returned %d\n",


@@ -489,17 +488,9 @@ void class_decref(struct obd_device *obd)
         CDEBUG(D_INFO, "Decref %s (%p) now %d\n", obd->obd_name, obd, refs);
 
         if ((refs == 1) && obd->obd_stopping) {
         CDEBUG(D_INFO, "Decref %s (%p) now %d\n", obd->obd_name, obd, refs);
 
         if ((refs == 1) && obd->obd_stopping) {

-                /* All exports (other than the self-export) have been
-                   destroyed; there should be no more in-progress ops
-                   by this point.*/
-                /* if we're not stopping, we didn't finish setup */
-                /* Precleanup stage 2,  do other type-specific
-                   cleanup requiring the self-export. */
-                err = obd_precleanup(obd, OBD_CLEANUP_SELF_EXP);
-                if (err)
-                        CERROR("Precleanup %s returned %d\n",
-                               obd->obd_name, err);
-                
+                /* All exports have been destroyed; there should
+                   be no more in-progress ops by this point.*/
+

                 spin_lock(&obd->obd_self_export->exp_lock);
                 obd->obd_self_export->exp_flags |=
                         (obd->obd_fail ? OBD_OPT_FAILOVER : 0) |
                 spin_lock(&obd->obd_self_export->exp_lock);
                 obd->obd_self_export->exp_flags |=
                         (obd->obd_fail ? OBD_OPT_FAILOVER : 0) |

diff --git a/lustre/obdfilter/filter.c b/lustre/obdfilter/filter.c
index f740f46..8f9ac20 100644 (file)
--- a/lustre/obdfilter/filter.c
+++ b/lustre/obdfilter/filter.c
@@ -2455,10 +2455,6 @@ static int filter_precleanup(struct obd_device *obd,
                 target_cleanup_recovery(obd);
                 rc = filter_llog_preclean(obd);
                 break;
                 target_cleanup_recovery(obd);
                 rc = filter_llog_preclean(obd);
                 break;

-        case OBD_CLEANUP_SELF_EXP:
-                break;
-        case OBD_CLEANUP_OBD:
-                break;

         }
         RETURN(rc);
 }
         }
         RETURN(rc);
 }

diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c
index 17c7e1d..24f7530 100644 (file)
--- a/lustre/osc/osc_request.c
+++ b/lustre/osc/osc_request.c
@@ -3989,11 +3989,7 @@ static int osc_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage)
                 if (rc != 0)
                         CERROR("failed to cleanup llogging subsystems\n");
                 break;
                 if (rc != 0)
                         CERROR("failed to cleanup llogging subsystems\n");
                 break;

-        }
-        case OBD_CLEANUP_SELF_EXP:
-                break;
-        case OBD_CLEANUP_OBD:
-                break;
+               }

         }
         RETURN(rc);
 }
         }
         RETURN(rc);
 }