From f01f2174bcc27c06c8cd5e64cb6e63452fe6c07b Mon Sep 17 00:00:00 2001 From: Nathaniel Clark Date: Mon, 16 Apr 2018 17:10:49 -0400 Subject: [PATCH] LU-10843 mgs: allow snapshot after MGS remount If the MGS is unmounted/mounted without the MDS reconnecting, the fsdb FSNAME-barrier would not be created. This change allows mgs_barrier_freeze (called from snapshot_create) to attempt the create the required fsdb. This adds a test to sanity-lsnapshot.sh for this issue. Test-Parameters: testlist=sanity-lsnapshot mgtfilesystemtype=zfs ostfilesystemtype=zfs mdtfilesystemtype=zfs combinedmdsmgs=false ostcount=2 standalonemgs=true Lustre-change: https://review.whamcloud.com/31779 Lustre-commit: 2d63d5bc74a5b2477d795bf9e9b0eba94f7fc19f Signed-off-by: Nathaniel Clark Change-Id: I2432cc0bdaddb07f024744065ca2ced77288fd7b Reviewed-by: Fan Yong Reviewed-by: Andreas Dilger Reviewed-by: Jian Yu Reviewed-by: Mike Pershin Signed-off-by: Minh Diep Reviewed-on: https://review.whamcloud.com/32519 Reviewed-by: Nathaniel Clark Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Jian Yu Reviewed-by: John L. Hammond --- lustre/mgs/mgs_barrier.c | 83 +++++++++++++++++++++++++++++++++------- lustre/mgs/mgs_internal.h | 3 ++ lustre/mgs/mgs_llog.c | 21 ++++++++-- lustre/tests/sanity-lsnapshot.sh | 35 +++++++++++++++++ 4 files changed, 125 insertions(+), 17 deletions(-) diff --git a/lustre/mgs/mgs_barrier.c b/lustre/mgs/mgs_barrier.c index e661063..7da6fb9 100644 --- a/lustre/mgs/mgs_barrier.c +++ b/lustre/mgs/mgs_barrier.c @@ -304,17 +304,36 @@ static int mgs_barrier_freeze(const struct lu_env *env, down_write(&mgs->mgs_barrier_rwsem); mutex_lock(&mgs->mgs_mutex); - fsdb = mgs_find_fsdb(mgs, name); - if (!fsdb) { + rc = mgs_find_or_make_fsdb_nolock(env, mgs, name, &fsdb); + if (rc) { mutex_unlock(&mgs->mgs_mutex); up_write(&mgs->mgs_barrier_rwsem); - - RETURN(-ENODEV); + RETURN(rc); } - if (unlikely(fsdb->fsdb_mdt_count == 0)) + if (unlikely(fsdb->fsdb_mdt_count == 0)) { mgs_barrier_bitmap_setup(mgs, fsdb, bc->bc_name); + /* fsdb was just created, ensure that fsdb_barrier_disabled is + * set correctly */ + if (fsdb->fsdb_mdt_count > 0) { + struct obd_export *exp; + struct obd_device *mgs_obd = mgs->mgs_obd; + + spin_lock(&mgs_obd->obd_dev_lock); + list_for_each_entry(exp, &mgs_obd->obd_exports, + exp_obd_chain) { + __u64 flags = exp_connect_flags(exp); + if (!!(flags & OBD_CONNECT_MDS_MDS) && + !(flags & OBD_CONNECT_BARRIER)) { + fsdb->fsdb_barrier_disabled = 1; + break; + } + } + spin_unlock(&mgs_obd->obd_dev_lock); + } + } + mutex_lock(&fsdb->fsdb_mutex); mutex_unlock(&mgs->mgs_mutex); @@ -445,17 +464,36 @@ static int mgs_barrier_thaw(const struct lu_env *env, down_write(&mgs->mgs_barrier_rwsem); mutex_lock(&mgs->mgs_mutex); - fsdb = mgs_find_fsdb(mgs, name); - if (!fsdb) { + rc = mgs_find_or_make_fsdb_nolock(env, mgs, name, &fsdb); + if (rc) { mutex_unlock(&mgs->mgs_mutex); up_write(&mgs->mgs_barrier_rwsem); - - RETURN(-ENODEV); + RETURN(rc); } - if (unlikely(fsdb->fsdb_mdt_count == 0)) + if (unlikely(fsdb->fsdb_mdt_count == 0)) { mgs_barrier_bitmap_setup(mgs, fsdb, bc->bc_name); + /* fsdb was just created, ensure that fsdb_barrier_disabled is + * set correctly */ + if (fsdb->fsdb_mdt_count > 0) { + struct obd_export *exp; + struct obd_device *mgs_obd = mgs->mgs_obd; + + spin_lock(&mgs_obd->obd_dev_lock); + list_for_each_entry(exp, &mgs_obd->obd_exports, + exp_obd_chain) { + __u64 flags = exp_connect_flags(exp); + if (!!(flags & OBD_CONNECT_MDS_MDS) && + !(flags & OBD_CONNECT_BARRIER)) { + fsdb->fsdb_barrier_disabled = 1; + break; + } + } + spin_unlock(&mgs_obd->obd_dev_lock); + } + } + mutex_lock(&fsdb->fsdb_mutex); mutex_unlock(&mgs->mgs_mutex); @@ -596,13 +634,32 @@ static int mgs_barrier_rescan(const struct lu_env *env, snprintf(name, sizeof(mgs_env_info(env)->mgi_fsname) - 1, "%s-%s", bc->bc_name, BARRIER_FILENAME); - b_fsdb = mgs_find_fsdb(mgs, name); - if (!b_fsdb) { + rc = mgs_find_or_make_fsdb_nolock(env, mgs, name, &b_fsdb); + if (rc) { mutex_unlock(&mgs->mgs_mutex); up_write(&mgs->mgs_barrier_rwsem); mgs_put_fsdb(mgs, c_fsdb); + RETURN(rc); + } - RETURN(-ENODEV); + if (unlikely(b_fsdb->fsdb_mdt_count == 0 && + c_fsdb->fsdb_mdt_count > 0)) { + /* fsdb was just created, ensure that fsdb_barrier_disabled is + * set correctly */ + struct obd_export *exp; + struct obd_device *mgs_obd = mgs->mgs_obd; + + spin_lock(&mgs_obd->obd_dev_lock); + list_for_each_entry(exp, &mgs_obd->obd_exports, + exp_obd_chain) { + __u64 flags = exp_connect_flags(exp); + if (!!(flags & OBD_CONNECT_MDS_MDS) && + !(flags & OBD_CONNECT_BARRIER)) { + b_fsdb->fsdb_barrier_disabled = 1; + break; + } + } + spin_unlock(&mgs_obd->obd_dev_lock); } mutex_lock(&b_fsdb->fsdb_mutex); diff --git a/lustre/mgs/mgs_internal.h b/lustre/mgs/mgs_internal.h index 28aa7db..5eb9ea7 100644 --- a/lustre/mgs/mgs_internal.h +++ b/lustre/mgs/mgs_internal.h @@ -201,6 +201,9 @@ int mgs_params_fsdb_setup(const struct lu_env *env, struct mgs_device *mgs); int mgs_params_fsdb_cleanup(const struct lu_env *env, struct mgs_device *mgs); int mgs_find_or_make_fsdb(const struct lu_env *env, struct mgs_device *mgs, char *name, struct fs_db **dbh); +int mgs_find_or_make_fsdb_nolock(const struct lu_env *env, + struct mgs_device *mgs, char *name, + struct fs_db **dbh); struct fs_db *mgs_find_fsdb(struct mgs_device *mgs, const char *fsname); void mgs_put_fsdb(struct mgs_device *mgs, struct fs_db *fsdb); int mgs_get_fsdb_srpc_from_llog(const struct lu_env *env, diff --git a/lustre/mgs/mgs_llog.c b/lustre/mgs/mgs_llog.c index 350ab00..d9d110c 100644 --- a/lustre/mgs/mgs_llog.c +++ b/lustre/mgs/mgs_llog.c @@ -541,14 +541,15 @@ int mgs_cleanup_fsdb_list(struct mgs_device *mgs) return 0; } -int mgs_find_or_make_fsdb(const struct lu_env *env, struct mgs_device *mgs, - char *name, struct fs_db **dbh) +/* The caller must hold mgs->mgs_mutex. */ +int mgs_find_or_make_fsdb_nolock(const struct lu_env *env, + struct mgs_device *mgs, + char *name, struct fs_db **dbh) { struct fs_db *fsdb; int rc = 0; ENTRY; - mutex_lock(&mgs->mgs_mutex); fsdb = mgs_find_fsdb(mgs, name); if (!fsdb) { fsdb = mgs_new_fsdb(env, mgs, name); @@ -557,7 +558,6 @@ int mgs_find_or_make_fsdb(const struct lu_env *env, struct mgs_device *mgs, CDEBUG(D_MGS, "Created new db: rc = %d\n", rc); } - mutex_unlock(&mgs->mgs_mutex); if (!rc) *dbh = fsdb; @@ -565,6 +565,19 @@ int mgs_find_or_make_fsdb(const struct lu_env *env, struct mgs_device *mgs, RETURN(rc); } +int mgs_find_or_make_fsdb(const struct lu_env *env, struct mgs_device *mgs, + char *name, struct fs_db **dbh) +{ + int rc; + ENTRY; + + mutex_lock(&mgs->mgs_mutex); + rc = mgs_find_or_make_fsdb_nolock(env, mgs, name, dbh); + mutex_unlock(&mgs->mgs_mutex); + + RETURN(rc); +} + /* 1 = index in use 0 = index unused -1= empty client log */ diff --git a/lustre/tests/sanity-lsnapshot.sh b/lustre/tests/sanity-lsnapshot.sh index 26fd232..8126799 100755 --- a/lustre/tests/sanity-lsnapshot.sh +++ b/lustre/tests/sanity-lsnapshot.sh @@ -330,6 +330,41 @@ test_3b() { } run_test 3b "modify snapshot without original filesystem mounted" +test_4() { # LU-10843 + combined_mgs_mds && skip "Combined MGS/MDS" && return + + local rcmd="$LCTL get_param ldlm.namespaces.MGS.resource_count" + + local exports=$(do_facet mgs "$LCTL get_param -n mgs.MGS.num_exports") + local rcount=$(do_facet mgs $rcmd) + + echo "Remount MGT" + stop mgs || error "stop mgs failed" + start mgs $(mgsdevname) $MGS_MOUNT_OPTS || error "start mgs failed" + + echo "Wait for all reconnects" + local CMD="$LCTL get_param -n mgs.MGS.num_exports" + wait_update_facet mgs "$CMD" $exports || + lss_err "(1) failed to export from mgs" + + wait_update_facet mgs "$rcmd" $rcount || + lss_err "(2) failed to reconnect mds" + + echo "Create lss_4_0" + lsnapshot_create -n lss_4_0 -c "'It is test_4'" || + lss_err "(3) Fail to create lss_4_0" + + echo "List lss_4_0" + lsnapshot_list -n lss_4_0 || + lss_err "(4) Fail to list lss_4_0" + + echo "Destroy lss_4_0" + lsnapshot_destroy -n lss_4_0 || + lss_err "(5) Fail to destroy lss_4_0" + +} +run_test 4 "create/delete snapshot after MGS remount" + lss_cleanup do_facet mgs $LCTL set_param debug=-snapshot do_nodes $(comma_list $(mdts_nodes)) $LCTL set_param debug=-snapshot -- 1.8.3.1