LU-10843 mgs: allow snapshot after MGS remount

If the MGS is unmounted/mounted without the MDS reconnecting,
the fsdb FSNAME-barrier would not be created.

This change allows mgs_barrier_freeze (called from snapshot_create)
to attempt the create the required fsdb.

This adds a test to sanity-lsnapshot.sh for this issue.

Test-Parameters: testlist=sanity-lsnapshot mgtfilesystemtype=zfs ostfilesystemtype=zfs mdtfilesystemtype=zfs combinedmdsmgs=false ostcount=2 standalonemgs=true
Signed-off-by: Nathaniel Clark <nathaniel.l.clark@intel.com>
Change-Id: I2432cc0bdaddb07f024744065ca2ced77288fd7b
Reviewed-on: https://review.whamcloud.com/31779
Tested-by: Jenkins
Reviewed-by: Fan Yong <fan.yong@intel.com>
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Reviewed-by: Jian Yu <jian.yu@intel.com>
Reviewed-by: Mike Pershin <mike.pershin@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lustre/mgs/mgs_barrier.c b/lustre/mgs/mgs_barrier.c
index 7dcb598..225bfa9 100644 (file)
--- a/lustre/mgs/mgs_barrier.c
+++ b/lustre/mgs/mgs_barrier.c
@@ -311,17 +311,36 @@ static int mgs_barrier_freeze(const struct lu_env *env,
        down_write(&mgs->mgs_barrier_rwsem);
        mutex_lock(&mgs->mgs_mutex);
 
        down_write(&mgs->mgs_barrier_rwsem);
        mutex_lock(&mgs->mgs_mutex);
 

-       fsdb = mgs_find_fsdb(mgs, name);
-       if (!fsdb) {
+       rc = mgs_find_or_make_fsdb_nolock(env, mgs, name, &fsdb);
+       if (rc) {

                mutex_unlock(&mgs->mgs_mutex);
                up_write(&mgs->mgs_barrier_rwsem);
                mutex_unlock(&mgs->mgs_mutex);
                up_write(&mgs->mgs_barrier_rwsem);

-
-               RETURN(-ENODEV);
+               RETURN(rc);

        }
 
        }
 

-       if (unlikely(fsdb->fsdb_mdt_count == 0))
+       if (unlikely(fsdb->fsdb_mdt_count == 0)) {

                mgs_barrier_bitmap_setup(mgs, fsdb, bc->bc_name);
 
                mgs_barrier_bitmap_setup(mgs, fsdb, bc->bc_name);
 

+               /* fsdb was just created, ensure that fsdb_barrier_disabled is
+                * set correctly */
+               if (fsdb->fsdb_mdt_count > 0) {
+                       struct obd_export *exp;
+                       struct obd_device *mgs_obd = mgs->mgs_obd;
+
+                       spin_lock(&mgs_obd->obd_dev_lock);
+                       list_for_each_entry(exp, &mgs_obd->obd_exports,
+                                           exp_obd_chain) {
+                               __u64 flags = exp_connect_flags(exp);
+                               if (!!(flags & OBD_CONNECT_MDS_MDS) &&
+                                   !(flags & OBD_CONNECT_BARRIER)) {
+                                       fsdb->fsdb_barrier_disabled = 1;
+                                       break;
+                               }
+                       }
+                       spin_unlock(&mgs_obd->obd_dev_lock);
+               }
+       }
+

        mutex_lock(&fsdb->fsdb_mutex);
        mutex_unlock(&mgs->mgs_mutex);
 
        mutex_lock(&fsdb->fsdb_mutex);
        mutex_unlock(&mgs->mgs_mutex);
 


@@ -450,17 +469,36 @@ static int mgs_barrier_thaw(const struct lu_env *env,
        down_write(&mgs->mgs_barrier_rwsem);
        mutex_lock(&mgs->mgs_mutex);
 
        down_write(&mgs->mgs_barrier_rwsem);
        mutex_lock(&mgs->mgs_mutex);
 

-       fsdb = mgs_find_fsdb(mgs, name);
-       if (!fsdb) {
+       rc = mgs_find_or_make_fsdb_nolock(env, mgs, name, &fsdb);
+       if (rc) {

                mutex_unlock(&mgs->mgs_mutex);
                up_write(&mgs->mgs_barrier_rwsem);
                mutex_unlock(&mgs->mgs_mutex);
                up_write(&mgs->mgs_barrier_rwsem);

-
-               RETURN(-ENODEV);
+               RETURN(rc);

        }
 
        }
 

-       if (unlikely(fsdb->fsdb_mdt_count == 0))
+       if (unlikely(fsdb->fsdb_mdt_count == 0)) {

                mgs_barrier_bitmap_setup(mgs, fsdb, bc->bc_name);
 
                mgs_barrier_bitmap_setup(mgs, fsdb, bc->bc_name);
 

+               /* fsdb was just created, ensure that fsdb_barrier_disabled is
+                * set correctly */
+               if (fsdb->fsdb_mdt_count > 0) {
+                       struct obd_export *exp;
+                       struct obd_device *mgs_obd = mgs->mgs_obd;
+
+                       spin_lock(&mgs_obd->obd_dev_lock);
+                       list_for_each_entry(exp, &mgs_obd->obd_exports,
+                                           exp_obd_chain) {
+                               __u64 flags = exp_connect_flags(exp);
+                               if (!!(flags & OBD_CONNECT_MDS_MDS) &&
+                                   !(flags & OBD_CONNECT_BARRIER)) {
+                                       fsdb->fsdb_barrier_disabled = 1;
+                                       break;
+                               }
+                       }
+                       spin_unlock(&mgs_obd->obd_dev_lock);
+               }
+       }
+

        mutex_lock(&fsdb->fsdb_mutex);
        mutex_unlock(&mgs->mgs_mutex);
 
        mutex_lock(&fsdb->fsdb_mutex);
        mutex_unlock(&mgs->mgs_mutex);
 


@@ -599,13 +637,32 @@ static int mgs_barrier_rescan(const struct lu_env *env,
 
        snprintf(name, sizeof(mgs_env_info(env)->mgi_fsname) - 1, "%s-%s",
                 bc->bc_name, BARRIER_FILENAME);
 
        snprintf(name, sizeof(mgs_env_info(env)->mgi_fsname) - 1, "%s-%s",
                 bc->bc_name, BARRIER_FILENAME);

-       b_fsdb = mgs_find_fsdb(mgs, name);
-       if (!b_fsdb) {
+       rc = mgs_find_or_make_fsdb_nolock(env, mgs, name, &b_fsdb);
+       if (rc) {

                mutex_unlock(&mgs->mgs_mutex);
                up_write(&mgs->mgs_barrier_rwsem);
                mgs_put_fsdb(mgs, c_fsdb);
                mutex_unlock(&mgs->mgs_mutex);
                up_write(&mgs->mgs_barrier_rwsem);
                mgs_put_fsdb(mgs, c_fsdb);

+               RETURN(rc);
+       }

 
 

-               RETURN(-ENODEV);
+       if (unlikely(b_fsdb->fsdb_mdt_count == 0 &&
+                    c_fsdb->fsdb_mdt_count > 0)) {
+               /* fsdb was just created, ensure that fsdb_barrier_disabled is
+                * set correctly */
+               struct obd_export *exp;
+               struct obd_device *mgs_obd = mgs->mgs_obd;
+
+               spin_lock(&mgs_obd->obd_dev_lock);
+               list_for_each_entry(exp, &mgs_obd->obd_exports,
+                                   exp_obd_chain) {
+                       __u64 flags = exp_connect_flags(exp);
+                       if (!!(flags & OBD_CONNECT_MDS_MDS) &&
+                           !(flags & OBD_CONNECT_BARRIER)) {
+                               b_fsdb->fsdb_barrier_disabled = 1;
+                               break;
+                       }
+               }
+               spin_unlock(&mgs_obd->obd_dev_lock);

        }
 
        mutex_lock(&b_fsdb->fsdb_mutex);
        }
 
        mutex_lock(&b_fsdb->fsdb_mutex);

diff --git a/lustre/mgs/mgs_internal.h b/lustre/mgs/mgs_internal.h
index eeb6555..d4de273 100644 (file)
--- a/lustre/mgs/mgs_internal.h
+++ b/lustre/mgs/mgs_internal.h
@@ -201,6 +201,9 @@ int mgs_params_fsdb_setup(const struct lu_env *env, struct mgs_device *mgs);
 int mgs_params_fsdb_cleanup(const struct lu_env *env, struct mgs_device *mgs);
 int mgs_find_or_make_fsdb(const struct lu_env *env, struct mgs_device *mgs,
                          char *name, struct fs_db **dbh);
 int mgs_params_fsdb_cleanup(const struct lu_env *env, struct mgs_device *mgs);
 int mgs_find_or_make_fsdb(const struct lu_env *env, struct mgs_device *mgs,
                          char *name, struct fs_db **dbh);

+int mgs_find_or_make_fsdb_nolock(const struct lu_env *env,
+                                 struct mgs_device *mgs, char *name,
+                                 struct fs_db **dbh);

 struct fs_db *mgs_find_fsdb(struct mgs_device *mgs, const char *fsname);
 void mgs_put_fsdb(struct mgs_device *mgs, struct fs_db *fsdb);
 int mgs_get_fsdb_srpc_from_llog(const struct lu_env *env,
 struct fs_db *mgs_find_fsdb(struct mgs_device *mgs, const char *fsname);
 void mgs_put_fsdb(struct mgs_device *mgs, struct fs_db *fsdb);
 int mgs_get_fsdb_srpc_from_llog(const struct lu_env *env,

diff --git a/lustre/mgs/mgs_llog.c b/lustre/mgs/mgs_llog.c
index 2a99e57..b3220e7 100644 (file)
--- a/lustre/mgs/mgs_llog.c
+++ b/lustre/mgs/mgs_llog.c
@@ -542,14 +542,15 @@ int mgs_cleanup_fsdb_list(struct mgs_device *mgs)
        return 0;
 }
 
        return 0;
 }
 

-int mgs_find_or_make_fsdb(const struct lu_env *env, struct mgs_device *mgs,
-                         char *name, struct fs_db **dbh)
+/* The caller must hold mgs->mgs_mutex. */
+int mgs_find_or_make_fsdb_nolock(const struct lu_env *env,
+                               struct mgs_device *mgs,
+                               char *name, struct fs_db **dbh)

 {
        struct fs_db *fsdb;
        int rc = 0;
        ENTRY;
 
 {
        struct fs_db *fsdb;
        int rc = 0;
        ENTRY;
 

-       mutex_lock(&mgs->mgs_mutex);

        fsdb = mgs_find_fsdb(mgs, name);
        if (!fsdb) {
                fsdb = mgs_new_fsdb(env, mgs, name);
        fsdb = mgs_find_fsdb(mgs, name);
        if (!fsdb) {
                fsdb = mgs_new_fsdb(env, mgs, name);


@@ -558,7 +559,6 @@ int mgs_find_or_make_fsdb(const struct lu_env *env, struct mgs_device *mgs,
 
                CDEBUG(D_MGS, "Created new db: rc = %d\n", rc);
        }
 
                CDEBUG(D_MGS, "Created new db: rc = %d\n", rc);
        }

-       mutex_unlock(&mgs->mgs_mutex);

 
        if (!rc)
                *dbh = fsdb;
 
        if (!rc)
                *dbh = fsdb;


@@ -566,6 +566,19 @@ int mgs_find_or_make_fsdb(const struct lu_env *env, struct mgs_device *mgs,
        RETURN(rc);
 }
 
        RETURN(rc);
 }
 

+int mgs_find_or_make_fsdb(const struct lu_env *env, struct mgs_device *mgs,
+                         char *name, struct fs_db **dbh)
+{
+       int rc;
+       ENTRY;
+
+       mutex_lock(&mgs->mgs_mutex);
+       rc = mgs_find_or_make_fsdb_nolock(env, mgs, name, dbh);
+       mutex_unlock(&mgs->mgs_mutex);
+
+       RETURN(rc);
+}
+

 /* 1 = index in use
    0 = index unused
    -1= empty client log */
 /* 1 = index in use
    0 = index unused
    -1= empty client log */

diff --git a/lustre/tests/sanity-lsnapshot.sh b/lustre/tests/sanity-lsnapshot.sh
index 26fd232..8126799 100755 (executable)
--- a/lustre/tests/sanity-lsnapshot.sh
+++ b/lustre/tests/sanity-lsnapshot.sh
@@ -330,6 +330,41 @@ test_3b() {
 }
 run_test 3b "modify snapshot without original filesystem mounted"
 
 }
 run_test 3b "modify snapshot without original filesystem mounted"
 

+test_4() { # LU-10843
+       combined_mgs_mds && skip "Combined MGS/MDS" && return
+
+       local rcmd="$LCTL get_param ldlm.namespaces.MGS.resource_count"
+
+       local exports=$(do_facet mgs "$LCTL get_param -n mgs.MGS.num_exports")
+       local rcount=$(do_facet mgs $rcmd)
+
+       echo "Remount MGT"
+       stop mgs || error "stop mgs failed"
+       start mgs $(mgsdevname) $MGS_MOUNT_OPTS || error "start mgs failed"
+
+       echo "Wait for all reconnects"
+       local CMD="$LCTL get_param -n mgs.MGS.num_exports"
+       wait_update_facet mgs "$CMD" $exports ||
+               lss_err "(1) failed to export from mgs"
+
+       wait_update_facet mgs "$rcmd" $rcount ||
+               lss_err "(2) failed to reconnect mds"
+
+       echo "Create lss_4_0"
+       lsnapshot_create -n lss_4_0 -c "'It is test_4'" ||
+               lss_err "(3) Fail to create lss_4_0"
+
+       echo "List lss_4_0"
+       lsnapshot_list -n lss_4_0 ||
+               lss_err "(4) Fail to list lss_4_0"
+
+       echo "Destroy lss_4_0"
+       lsnapshot_destroy -n lss_4_0 ||
+               lss_err "(5) Fail to destroy lss_4_0"
+
+}
+run_test 4 "create/delete snapshot after MGS remount"
+

 lss_cleanup
 do_facet mgs $LCTL set_param debug=-snapshot
 do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param debug=-snapshot
 lss_cleanup
 do_facet mgs $LCTL set_param debug=-snapshot
 do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param debug=-snapshot