From bfa1dbc969df6e9e10579fdb30ab653835463bd2 Mon Sep 17 00:00:00 2001 From: Fan Yong Date: Tue, 27 Sep 2016 16:30:50 +0800 Subject: [PATCH] LU-8900 mgs: use reference count for fs_db That will prevent the in-using 'fs_db' being freed/erased by others. Then the user (in subsequent patches) can hold the 'fs_db' for a long time without holding related lock. Signed-off-by: Fan Yong Change-Id: Icf5d548a2c51548aae2c05b1b34f003e725f4e02 Reviewed-on: https://review.whamcloud.com/24415 Tested-by: Jenkins Tested-by: Maloo Reviewed-by: Andreas Dilger Reviewed-by: Niu Yawei Reviewed-by: Oleg Drokin --- lustre/mgs/mgs_handler.c | 26 +-- lustre/mgs/mgs_internal.h | 12 +- lustre/mgs/mgs_llog.c | 426 +++++++++++++++++++++++++++------------------- lustre/mgs/mgs_nids.c | 84 +++++---- 4 files changed, 316 insertions(+), 232 deletions(-) diff --git a/lustre/mgs/mgs_handler.c b/lustre/mgs/mgs_handler.c index c9bcf21..a31e796 100644 --- a/lustre/mgs/mgs_handler.c +++ b/lustre/mgs/mgs_handler.c @@ -338,12 +338,12 @@ static int mgs_check_failover_reg(struct mgs_target_info *mti) /* Called whenever a target starts up. Flags indicate first connect, etc. */ static int mgs_target_reg(struct tgt_session_info *tsi) { - struct obd_device *obd = tsi->tsi_exp->exp_obd; - struct mgs_device *mgs = exp2mgs_dev(tsi->tsi_exp); - struct mgs_target_info *mti, *rep_mti; - struct fs_db *fsdb; - int opc; - int rc = 0; + struct obd_device *obd = tsi->tsi_exp->exp_obd; + struct mgs_device *mgs = exp2mgs_dev(tsi->tsi_exp); + struct mgs_target_info *mti, *rep_mti; + struct fs_db *fsdb = NULL; + int opc; + int rc = 0; ENTRY; @@ -482,6 +482,8 @@ out_nolock: /* Flush logs to disk */ dt_sync(tsi->tsi_env, mgs->mgs_bottom); + if (fsdb) + mgs_put_fsdb(mgs, fsdb); RETURN(rc); } @@ -783,11 +785,13 @@ static int mgs_iocontrol_nodemap(const struct lu_env *env, /* revoke nodemap lock */ rc = mgs_find_or_make_fsdb(env, mgs, LUSTRE_NODEMAP_NAME, &fsdb); - if (rc < 0) + if (rc < 0) { CWARN("%s: cannot make nodemap fsdb: rc = %d\n", mgs->mgs_obd->obd_name, rc); - else + } else { mgs_revoke_lock(mgs, fsdb, CONFIG_T_NODEMAP); + mgs_put_fsdb(mgs, fsdb); + } out_lcfg: OBD_FREE(lcfg, data->ioc_plen1); @@ -1115,8 +1119,6 @@ static int mgs_init0(const struct lu_env *env, struct mgs_device *mgs, struct obd_device *obd; struct lustre_mount_info *lmi; struct llog_ctxt *ctxt; - struct fs_db *fsdb = NULL; - struct fs_db *fsdb_srpc = NULL; int rc; ENTRY; @@ -1191,14 +1193,14 @@ static int mgs_init0(const struct lu_env *env, struct mgs_device *mgs, /* Setup params fsdb and log, so that other servers can make a local * copy successfully when they are mounted. See LU-4783 */ - rc = mgs_params_fsdb_setup(env, mgs, fsdb); + rc = mgs_params_fsdb_setup(env, mgs); if (rc) /* params fsdb and log can be setup later */ CERROR("%s: %s fsdb and log setup failed: rc = %d\n", obd->obd_name, PARAMS_FILENAME, rc); /* Setup _mgs fsdb, useful for srpc */ - mgs__mgs_fsdb_setup(env, mgs, fsdb_srpc); + mgs__mgs_fsdb_setup(env, mgs); ptlrpc_init_client(LDLM_CB_REQUEST_PORTAL, LDLM_CB_REPLY_PORTAL, "mgs_ldlm_client", &obd->obd_ldlm_client); diff --git a/lustre/mgs/mgs_internal.h b/lustre/mgs/mgs_internal.h index 1e81522..7564cfd 100644 --- a/lustre/mgs/mgs_internal.h +++ b/lustre/mgs/mgs_internal.h @@ -122,6 +122,7 @@ struct fs_db { struct mutex fsdb_mutex; void *fsdb_ost_index_map; /* bitmap of used indicies */ void *fsdb_mdt_index_map; /* bitmap of used indicies */ + atomic_t fsdb_ref; int fsdb_mdt_count; char *fsdb_clilov; /* COMPAT_146 client lov name */ char *fsdb_clilmv; @@ -185,14 +186,13 @@ struct mgs_object { int mgs_init_fsdb_list(struct mgs_device *mgs); int mgs_cleanup_fsdb_list(struct mgs_device *mgs); -int mgs__mgs_fsdb_setup(const struct lu_env *env, struct mgs_device *mgs, - struct fs_db *fsdb); -int mgs_params_fsdb_setup(const struct lu_env *env, struct mgs_device *mgs, - struct fs_db *fsdb); +int mgs__mgs_fsdb_setup(const struct lu_env *env, struct mgs_device *mgs); +int mgs_params_fsdb_setup(const struct lu_env *env, struct mgs_device *mgs); int mgs_params_fsdb_cleanup(const struct lu_env *env, struct mgs_device *mgs); int mgs_find_or_make_fsdb(const struct lu_env *env, struct mgs_device *mgs, char *name, struct fs_db **dbh); -struct fs_db *mgs_find_fsdb(struct mgs_device *mgs, char *fsname); +struct fs_db *mgs_find_fsdb(struct mgs_device *mgs, const char *fsname); +void mgs_put_fsdb(struct mgs_device *mgs, struct fs_db *fsdb); int mgs_get_fsdb_srpc_from_llog(const struct lu_env *env, struct mgs_device *mgs, struct fs_db *fsdb); int mgs_check_index(const struct lu_env *env, struct mgs_device *mgs, @@ -204,7 +204,7 @@ int mgs_replace_nids(const struct lu_env *env, struct mgs_device *mgs, int mgs_erase_log(const struct lu_env *env, struct mgs_device *mgs, char *name); int mgs_erase_logs(const struct lu_env *env, struct mgs_device *mgs, - char *fsname); + const char *fsname); int mgs_setparam(const struct lu_env *env, struct mgs_device *mgs, struct lustre_cfg *lcfg, char *fsname); int mgs_list_logs(const struct lu_env *env, struct mgs_device *mgs, diff --git a/lustre/mgs/mgs_llog.c b/lustre/mgs/mgs_llog.c index 5d9c535..ec780ff 100644 --- a/lustre/mgs/mgs_llog.c +++ b/lustre/mgs/mgs_llog.c @@ -319,98 +319,177 @@ static void mgs_free_fsdb_srpc(struct fs_db *fsdb) sptlrpc_rule_set_free(&fsdb->fsdb_srpc_gen); } -struct fs_db *mgs_find_fsdb(struct mgs_device *mgs, char *fsname) +static void mgs_unlink_fsdb(struct mgs_device *mgs, struct fs_db *fsdb) { - struct fs_db *fsdb; + mutex_lock(&mgs->mgs_mutex); + if (likely(!list_empty(&fsdb->fsdb_list))) { + LASSERTF(atomic_read(&fsdb->fsdb_ref) >= 2, + "Invalid ref %d on %s\n", + atomic_read(&fsdb->fsdb_ref), + fsdb->fsdb_name); + + list_del_init(&fsdb->fsdb_list); + /* Drop the reference on the list.*/ + mgs_put_fsdb(mgs, fsdb); + } + mutex_unlock(&mgs->mgs_mutex); +} + +/* The caller must hold mgs->mgs_mutex. */ +static inline struct fs_db * +mgs_find_fsdb_noref(struct mgs_device *mgs, const char *fsname) +{ + struct fs_db *fsdb; struct list_head *tmp; list_for_each(tmp, &mgs->mgs_fs_db_list) { fsdb = list_entry(tmp, struct fs_db, fsdb_list); - if (strcmp(fsdb->fsdb_name, fsname) == 0) - return fsdb; - } - return NULL; + if (strcmp(fsdb->fsdb_name, fsname) == 0) + return fsdb; + } + + return NULL; +} + +/* The caller must hold mgs->mgs_mutex. */ +static void mgs_remove_fsdb_by_name(struct mgs_device *mgs, const char *name) +{ + struct fs_db *fsdb; + + fsdb = mgs_find_fsdb_noref(mgs, name); + if (fsdb) { + list_del_init(&fsdb->fsdb_list); + /* Drop the reference on the list.*/ + mgs_put_fsdb(mgs, fsdb); + } +} + +/* The caller must hold mgs->mgs_mutex. */ +struct fs_db *mgs_find_fsdb(struct mgs_device *mgs, const char *fsname) +{ + struct fs_db *fsdb; + + fsdb = mgs_find_fsdb_noref(mgs, fsname); + if (fsdb) + atomic_inc(&fsdb->fsdb_ref); + + return fsdb; } -/* caller must hold the mgs->mgs_fs_db_lock */ +/* The caller must hold mgs->mgs_mutex. */ static struct fs_db *mgs_new_fsdb(const struct lu_env *env, struct mgs_device *mgs, char *fsname) { - struct fs_db *fsdb; - int rc; - ENTRY; + struct fs_db *fsdb; + int rc; + ENTRY; + + if (strlen(fsname) >= sizeof(fsdb->fsdb_name)) { + CERROR("fsname %s is too long\n", fsname); - if (strlen(fsname) >= sizeof(fsdb->fsdb_name)) { - CERROR("fsname %s is too long\n", fsname); RETURN(ERR_PTR(-EINVAL)); - } + } - OBD_ALLOC_PTR(fsdb); - if (!fsdb) + OBD_ALLOC_PTR(fsdb); + if (!fsdb) RETURN(ERR_PTR(-ENOMEM)); - strcpy(fsdb->fsdb_name, fsname); + strncpy(fsdb->fsdb_name, fsname, sizeof(fsdb->fsdb_name)); mutex_init(&fsdb->fsdb_mutex); + INIT_LIST_HEAD(&fsdb->fsdb_list); set_bit(FSDB_UDESC, &fsdb->fsdb_flags); fsdb->fsdb_gen = 1; + INIT_LIST_HEAD(&fsdb->fsdb_clients); + atomic_set(&fsdb->fsdb_notify_phase, 0); + init_waitqueue_head(&fsdb->fsdb_notify_waitq); + init_completion(&fsdb->fsdb_notify_comp); - if (strcmp(fsname, MGSSELF_NAME) == 0) { + if (strcmp(fsname, MGSSELF_NAME) == 0) { set_bit(FSDB_MGS_SELF, &fsdb->fsdb_flags); fsdb->fsdb_mgs = mgs; - } else { - OBD_ALLOC(fsdb->fsdb_ost_index_map, INDEX_MAP_SIZE); - OBD_ALLOC(fsdb->fsdb_mdt_index_map, INDEX_MAP_SIZE); - if (!fsdb->fsdb_ost_index_map || !fsdb->fsdb_mdt_index_map) { - CERROR("No memory for index maps\n"); + } else { + OBD_ALLOC(fsdb->fsdb_mdt_index_map, INDEX_MAP_SIZE); + if (!fsdb->fsdb_mdt_index_map) { + CERROR("No memory for MDT index maps\n"); + GOTO(err, rc = -ENOMEM); - } + } - rc = name_create(&fsdb->fsdb_clilov, fsname, "-clilov"); - if (rc) - GOTO(err, rc); - rc = name_create(&fsdb->fsdb_clilmv, fsname, "-clilmv"); - if (rc) - GOTO(err, rc); + OBD_ALLOC(fsdb->fsdb_ost_index_map, INDEX_MAP_SIZE); + if (!fsdb->fsdb_ost_index_map) { + CERROR("No memory for OST index maps\n"); - /* initialise data for NID table */ - mgs_ir_init_fs(env, mgs, fsdb); + GOTO(err, rc = -ENOMEM); + } + + rc = name_create(&fsdb->fsdb_clilov, fsname, "-clilov"); + if (rc) + GOTO(err, rc); + rc = name_create(&fsdb->fsdb_clilmv, fsname, "-clilmv"); + if (rc) + GOTO(err, rc); + + /* initialise data for NID table */ + mgs_ir_init_fs(env, mgs, fsdb); lproc_mgs_add_live(mgs, fsdb); - } + } + + if (!test_bit(FSDB_MGS_SELF, &fsdb->fsdb_flags)) { + /* populate the db from the client llog */ + rc = mgs_get_fsdb_from_llog(env, mgs, fsdb); + if (rc) { + CERROR("Can't get db from client log %d\n", rc); + + GOTO(err, rc); + } + } + + /* populate srpc rules from params llog */ + rc = mgs_get_fsdb_srpc_from_llog(env, mgs, fsdb); + if (rc) { + CERROR("Can't get db from params log %d\n", rc); + + GOTO(err, rc); + } + /* One ref is for the fsdb on the list. + * The other ref is for the caller. */ + atomic_set(&fsdb->fsdb_ref, 2); list_add(&fsdb->fsdb_list, &mgs->mgs_fs_db_list); - RETURN(fsdb); + RETURN(fsdb); + err: - if (fsdb->fsdb_ost_index_map) - OBD_FREE(fsdb->fsdb_ost_index_map, INDEX_MAP_SIZE); - if (fsdb->fsdb_mdt_index_map) - OBD_FREE(fsdb->fsdb_mdt_index_map, INDEX_MAP_SIZE); - name_destroy(&fsdb->fsdb_clilov); - name_destroy(&fsdb->fsdb_clilmv); - OBD_FREE_PTR(fsdb); + atomic_set(&fsdb->fsdb_ref, 1); + mgs_put_fsdb(mgs, fsdb); + RETURN(ERR_PTR(rc)); } static void mgs_free_fsdb(struct mgs_device *mgs, struct fs_db *fsdb) { - /* wait for anyone with the sem */ - mutex_lock(&fsdb->fsdb_mutex); + LASSERT(list_empty(&fsdb->fsdb_list)); + lproc_mgs_del_live(mgs, fsdb); - list_del(&fsdb->fsdb_list); - /* deinitialize fsr */ + /* deinitialize fsr */ mgs_ir_fini_fs(mgs, fsdb); - if (fsdb->fsdb_ost_index_map) - OBD_FREE(fsdb->fsdb_ost_index_map, INDEX_MAP_SIZE); - if (fsdb->fsdb_mdt_index_map) - OBD_FREE(fsdb->fsdb_mdt_index_map, INDEX_MAP_SIZE); - name_destroy(&fsdb->fsdb_clilov); - name_destroy(&fsdb->fsdb_clilmv); - mgs_free_fsdb_srpc(fsdb); - mutex_unlock(&fsdb->fsdb_mutex); - OBD_FREE_PTR(fsdb); + if (fsdb->fsdb_ost_index_map) + OBD_FREE(fsdb->fsdb_ost_index_map, INDEX_MAP_SIZE); + if (fsdb->fsdb_mdt_index_map) + OBD_FREE(fsdb->fsdb_mdt_index_map, INDEX_MAP_SIZE); + name_destroy(&fsdb->fsdb_clilov); + name_destroy(&fsdb->fsdb_clilmv); + mgs_free_fsdb_srpc(fsdb); + OBD_FREE_PTR(fsdb); +} + +void mgs_put_fsdb(struct mgs_device *mgs, struct fs_db *fsdb) +{ + if (atomic_dec_and_test(&fsdb->fsdb_ref)) + mgs_free_fsdb(mgs, fsdb); } int mgs_init_fsdb_list(struct mgs_device *mgs) @@ -427,62 +506,35 @@ int mgs_cleanup_fsdb_list(struct mgs_device *mgs) mutex_lock(&mgs->mgs_mutex); list_for_each_safe(tmp, tmp2, &mgs->mgs_fs_db_list) { fsdb = list_entry(tmp, struct fs_db, fsdb_list); - mgs_free_fsdb(mgs, fsdb); + list_del_init(&fsdb->fsdb_list); + mgs_put_fsdb(mgs, fsdb); } mutex_unlock(&mgs->mgs_mutex); return 0; } -int mgs_find_or_make_fsdb(const struct lu_env *env, - struct mgs_device *mgs, char *name, - struct fs_db **dbh) +int mgs_find_or_make_fsdb(const struct lu_env *env, struct mgs_device *mgs, + char *name, struct fs_db **dbh) { - struct fs_db *fsdb; - int rc = 0; - + struct fs_db *fsdb; + int rc = 0; ENTRY; + mutex_lock(&mgs->mgs_mutex); fsdb = mgs_find_fsdb(mgs, name); - if (fsdb) { - mutex_unlock(&mgs->mgs_mutex); - *dbh = fsdb; - RETURN(0); - } + if (!fsdb) { + fsdb = mgs_new_fsdb(env, mgs, name); + if (IS_ERR(fsdb)) + rc = PTR_ERR(fsdb); - CDEBUG(D_MGS, "Creating new db\n"); - fsdb = mgs_new_fsdb(env, mgs, name); - /* lock fsdb_mutex until the db is loaded from llogs */ - if (!IS_ERR(fsdb)) - mutex_lock(&fsdb->fsdb_mutex); + CDEBUG(D_MGS, "Created new db: rc = %d\n", rc); + } mutex_unlock(&mgs->mgs_mutex); - if (IS_ERR(fsdb)) - RETURN(PTR_ERR(fsdb)); - if (!test_bit(FSDB_MGS_SELF, &fsdb->fsdb_flags)) { - /* populate the db from the client llog */ - rc = mgs_get_fsdb_from_llog(env, mgs, fsdb); - if (rc) { - CERROR("Can't get db from client log %d\n", rc); - GOTO(out_free, rc); - } - } - - /* populate srpc rules from params llog */ - rc = mgs_get_fsdb_srpc_from_llog(env, mgs, fsdb); - if (rc) { - CERROR("Can't get db from params log %d\n", rc); - GOTO(out_free, rc); - } - - mutex_unlock(&fsdb->fsdb_mutex); - *dbh = fsdb; - - RETURN(0); + if (!rc) + *dbh = fsdb; -out_free: - mutex_unlock(&fsdb->fsdb_mutex); - mgs_free_fsdb(mgs, fsdb); - return rc; + RETURN(rc); } /* 1 = index in use @@ -506,18 +558,23 @@ int mgs_check_index(const struct lu_env *env, } if (test_bit(FSDB_LOG_EMPTY, &fsdb->fsdb_flags)) - RETURN(-1); + GOTO(out, rc = -1); - if (mti->mti_flags & LDD_F_SV_TYPE_OST) - imap = fsdb->fsdb_ost_index_map; - else if (mti->mti_flags & LDD_F_SV_TYPE_MDT) - imap = fsdb->fsdb_mdt_index_map; - else - RETURN(-EINVAL); + if (mti->mti_flags & LDD_F_SV_TYPE_OST) + imap = fsdb->fsdb_ost_index_map; + else if (mti->mti_flags & LDD_F_SV_TYPE_MDT) + imap = fsdb->fsdb_mdt_index_map; + else + GOTO(out, rc = -EINVAL); if (test_bit(mti->mti_stripe_index, imap)) - RETURN(1); - RETURN(0); + GOTO(out, rc = 1); + + GOTO(out, rc = 0); + +out: + mgs_put_fsdb(mgs, fsdb); + return rc; } static __inline__ int next_index(void *index_map, int map_len) @@ -595,20 +652,21 @@ static int mgs_set_index(const struct lu_env *env, set_bit(mti->mti_stripe_index, imap); clear_bit(FSDB_LOG_EMPTY, &fsdb->fsdb_flags); - mutex_unlock(&fsdb->fsdb_mutex); if (server_make_name(mti->mti_flags & ~(LDD_F_VIRGIN | LDD_F_WRITECONF), mti->mti_stripe_index, mti->mti_fsname, mti->mti_svname)) { CERROR("unknown server type %#x\n", mti->mti_flags); - return -EINVAL; + GOTO(out_up, rc = -EINVAL); } - CDEBUG(D_MGS, "Set index for %s to %d\n", mti->mti_svname, - mti->mti_stripe_index); + CDEBUG(D_MGS, "Set index for %s to %d\n", mti->mti_svname, + mti->mti_stripe_index); + + GOTO(out_up, rc = 0); - RETURN(0); out_up: mutex_unlock(&fsdb->fsdb_mutex); + mgs_put_fsdb(mgs, fsdb); return rc; } @@ -1333,7 +1391,7 @@ int mgs_replace_nids(const struct lu_env *env, int rc; __u32 index; char *logname; - struct fs_db *fsdb; + struct fs_db *fsdb = NULL; unsigned int i; int conn_state; struct obd_device *mgs_obd = mgs->mgs_obd; @@ -1388,6 +1446,9 @@ out: mgs_obd->obd_no_conn = conn_state; spin_unlock(&mgs_obd->obd_dev_lock); + if (fsdb) + mgs_put_fsdb(mgs, fsdb); + RETURN(rc); } @@ -3940,9 +4001,9 @@ int mgs_erase_log(const struct lu_env *env, struct mgs_device *mgs, char *name) } /* erase all logs for the given fs */ -int mgs_erase_logs(const struct lu_env *env, struct mgs_device *mgs, char *fsname) +int mgs_erase_logs(const struct lu_env *env, struct mgs_device *mgs, + const char *fsname) { - struct fs_db *fsdb; struct list_head log_list; struct mgs_direntry *dirent, *n; int rc, len = strlen(fsname); @@ -3955,12 +4016,8 @@ int mgs_erase_logs(const struct lu_env *env, struct mgs_device *mgs, char *fsnam RETURN(rc); mutex_lock(&mgs->mgs_mutex); - - /* Delete the fs db */ - fsdb = mgs_find_fsdb(mgs, fsname); - if (fsdb) - mgs_free_fsdb(mgs, fsdb); - + /* Delete the fs db */ + mgs_remove_fsdb_by_name(mgs, fsname); mutex_unlock(&mgs->mgs_mutex); list_for_each_entry_safe(dirent, n, &log_list, mde_list) { @@ -3977,7 +4034,7 @@ int mgs_erase_logs(const struct lu_env *env, struct mgs_device *mgs, char *fsnam mgs_direntry_free(dirent); } - RETURN(rc); + RETURN(rc); } /* list all logs for the given fs */ @@ -4040,33 +4097,36 @@ static void print_lustre_cfg(struct lustre_cfg *lcfg) /* Setup _mgs fsdb and log */ -int mgs__mgs_fsdb_setup(const struct lu_env *env, struct mgs_device *mgs, - struct fs_db *fsdb) +int mgs__mgs_fsdb_setup(const struct lu_env *env, struct mgs_device *mgs) { - int rc; + struct fs_db *fsdb = NULL; + int rc; ENTRY; rc = mgs_find_or_make_fsdb(env, mgs, MGSSELF_NAME, &fsdb); + if (!rc) + mgs_put_fsdb(mgs, fsdb); RETURN(rc); } /* Setup params fsdb and log */ -int mgs_params_fsdb_setup(const struct lu_env *env, struct mgs_device *mgs, - struct fs_db *fsdb) +int mgs_params_fsdb_setup(const struct lu_env *env, struct mgs_device *mgs) { - struct llog_handle *params_llh = NULL; - int rc; + struct fs_db *fsdb = NULL; + struct llog_handle *params_llh = NULL; + int rc; ENTRY; rc = mgs_find_or_make_fsdb(env, mgs, PARAMS_FILENAME, &fsdb); - if (fsdb != NULL) { + if (!rc) { mutex_lock(&fsdb->fsdb_mutex); rc = record_start_log(env, mgs, ¶ms_llh, PARAMS_FILENAME); - if (rc == 0) + if (!rc) rc = record_end_log(env, ¶ms_llh); mutex_unlock(&fsdb->fsdb_mutex); + mgs_put_fsdb(mgs, fsdb); } RETURN(rc); @@ -4086,13 +4146,14 @@ int mgs_params_fsdb_cleanup(const struct lu_env *env, struct mgs_device *mgs) int mgs_setparam(const struct lu_env *env, struct mgs_device *mgs, struct lustre_cfg *lcfg, char *fsname) { - struct fs_db *fsdb; - struct mgs_target_info *mti; - char *devname, *param; + struct fs_db *fsdb = NULL; + struct mgs_target_info *mti = NULL; + char *devname, *param; char *ptr; const char *tmp; __u32 index; int rc = 0; + bool free = false; ENTRY; print_lustre_cfg(lcfg); @@ -4136,16 +4197,16 @@ int mgs_setparam(const struct lu_env *env, struct mgs_device *mgs, if (lcfg->lcfg_command != LCFG_SET_PARAM && !test_bit(FSDB_MGS_SELF, &fsdb->fsdb_flags) && test_bit(FSDB_LOG_EMPTY, &fsdb->fsdb_flags)) { - CERROR("No filesystem targets for %s. cfg_device from lctl " - "is '%s'\n", fsname, devname); - mgs_free_fsdb(mgs, fsdb); - RETURN(-EINVAL); - } + CERROR("No filesystem targets for %s. cfg_device from lctl " + "is '%s'\n", fsname, devname); + free = true; + GOTO(out, rc = -EINVAL); + } - /* Create a fake mti to hold everything */ - OBD_ALLOC_PTR(mti); - if (!mti) - GOTO(out, rc = -ENOMEM); + /* Create a fake mti to hold everything */ + OBD_ALLOC_PTR(mti); + if (!mti) + GOTO(out, rc = -ENOMEM); if (strlcpy(mti->mti_fsname, fsname, sizeof(mti->mti_fsname)) >= sizeof(mti->mti_fsname)) GOTO(out, rc = -E2BIG); @@ -4185,8 +4246,16 @@ int mgs_setparam(const struct lu_env *env, struct mgs_device *mgs, } out: - OBD_FREE_PTR(mti); - RETURN(rc); + if (mti) + OBD_FREE_PTR(mti); + + if (fsdb) { + if (free) + mgs_unlink_fsdb(mgs, fsdb); + mgs_put_fsdb(mgs, fsdb); + } + + RETURN(rc); } static int mgs_write_log_pool(const struct lu_env *env, @@ -4310,18 +4379,20 @@ int mgs_pool_cmd(const struct lu_env *env, struct mgs_device *mgs, int label_sz; struct mgs_target_info *mti = NULL; bool checked = false; - int rc, i; - ENTRY; + bool locked = false; + bool free = false; + int rc, i; + ENTRY; rc = mgs_find_or_make_fsdb(env, mgs, fsname, &fsdb); - if (rc) { - CERROR("Can't get db for %s\n", fsname); - RETURN(rc); - } + if (rc) { + CERROR("Can't get db for %s\n", fsname); + RETURN(rc); + } if (test_bit(FSDB_LOG_EMPTY, &fsdb->fsdb_flags)) { - CERROR("%s is not defined\n", fsname); - mgs_free_fsdb(mgs, fsdb); - RETURN(-EINVAL); + CERROR("%s is not defined\n", fsname); + free = true; + GOTO(out_fsdb, rc = -EINVAL); } label_sz = 10 + strlen(fsname) + strlen(poolname); @@ -4338,8 +4409,8 @@ int mgs_pool_cmd(const struct lu_env *env, struct mgs_device *mgs, } OBD_ALLOC(label, label_sz); - if (label == NULL) - RETURN(-ENOMEM); + if (!label) + GOTO(out_fsdb, rc = -ENOMEM); switch(cmd) { case LCFG_POOL_NEW: @@ -4378,15 +4449,14 @@ int mgs_pool_cmd(const struct lu_env *env, struct mgs_device *mgs, strncpy(mti->mti_svname, "lov pool", sizeof(mti->mti_svname)); mutex_lock(&fsdb->fsdb_mutex); - /* write pool def to all MDT logs */ - for (i = 0; i < INDEX_MAP_SIZE * 8; i++) { + locked = true; + /* write pool def to all MDT logs */ + for (i = 0; i < INDEX_MAP_SIZE * 8; i++) { if (test_bit(i, fsdb->fsdb_mdt_index_map)) { rc = name_create_mdt_and_lov(&logname, &lovname, fsdb, i); - if (rc) { - mutex_unlock(&fsdb->fsdb_mutex); + if (rc) GOTO(out_mti, rc); - } if (!checked && (canceled_label == NULL)) { rc = mgs_check_marker(env, mgs, fsdb, mti, @@ -4394,7 +4464,6 @@ int mgs_pool_cmd(const struct lu_env *env, struct mgs_device *mgs, if (rc) { name_destroy(&logname); name_destroy(&lovname); - mutex_unlock(&fsdb->fsdb_mutex); GOTO(out_mti, rc = (rc == LLOG_PROC_BREAK ? -EEXIST : rc)); @@ -4411,27 +4480,22 @@ int mgs_pool_cmd(const struct lu_env *env, struct mgs_device *mgs, fsdb, lovname, cmd, fsname, poolname, ostname, label); - name_destroy(&logname); - name_destroy(&lovname); - if (rc) { - mutex_unlock(&fsdb->fsdb_mutex); + name_destroy(&logname); + name_destroy(&lovname); + if (rc) GOTO(out_mti, rc); - } - } - } + } + } rc = name_create(&logname, fsname, "-client"); - if (rc) { - mutex_unlock(&fsdb->fsdb_mutex); + if (rc) GOTO(out_mti, rc); - } if (!checked && (canceled_label == NULL)) { rc = mgs_check_marker(env, mgs, fsdb, mti, logname, fsdb->fsdb_clilov, label); if (rc) { name_destroy(&logname); - mutex_unlock(&fsdb->fsdb_mutex); GOTO(out_mti, rc = (rc == LLOG_PROC_BREAK ? -EEXIST : rc)); } @@ -4440,7 +4504,6 @@ int mgs_pool_cmd(const struct lu_env *env, struct mgs_device *mgs, rc = mgs_modify(env, mgs, fsdb, mti, logname, fsdb->fsdb_clilov, canceled_label, CM_SKIP); if (rc < 0) { - mutex_unlock(&fsdb->fsdb_mutex); name_destroy(&logname); GOTO(out_mti, rc); } @@ -4449,18 +4512,27 @@ int mgs_pool_cmd(const struct lu_env *env, struct mgs_device *mgs, rc = mgs_write_log_pool(env, mgs, logname, fsdb, fsdb->fsdb_clilov, cmd, fsname, poolname, ostname, label); mutex_unlock(&fsdb->fsdb_mutex); + locked = false; name_destroy(&logname); - /* request for update */ + /* request for update */ mgs_revoke_lock(mgs, fsdb, CONFIG_T_CONFIG); - EXIT; + GOTO(out_mti, rc); + out_mti: - if (mti != NULL) - OBD_FREE_PTR(mti); + if (locked) + mutex_unlock(&fsdb->fsdb_mutex); + if (mti != NULL) + OBD_FREE_PTR(mti); out_cancel: if (canceled_label != NULL) OBD_FREE(canceled_label, label_sz); out_label: OBD_FREE(label, label_sz); - return rc; +out_fsdb: + if (free) + mgs_unlink_fsdb(mgs, fsdb); + mgs_put_fsdb(mgs, fsdb); + + return rc; } diff --git a/lustre/mgs/mgs_nids.c b/lustre/mgs/mgs_nids.c index b4ce9431..9338d80 100644 --- a/lustre/mgs/mgs_nids.c +++ b/lustre/mgs/mgs_nids.c @@ -461,14 +461,8 @@ int mgs_ir_init_fs(const struct lu_env *env, struct mgs_device *mgs, mgs->mgs_start_time + ir_timeout)) fsdb->fsdb_ir_state = IR_STARTUP; fsdb->fsdb_nonir_clients = 0; - INIT_LIST_HEAD(&fsdb->fsdb_clients); - /* start notify thread */ fsdb->fsdb_mgs = mgs; - atomic_set(&fsdb->fsdb_notify_phase, 0); - init_waitqueue_head(&fsdb->fsdb_notify_waitq); - init_completion(&fsdb->fsdb_notify_comp); - task = kthread_run(mgs_ir_notify, fsdb, "mgs_%s_notify", fsdb->fsdb_name); if (!IS_ERR(task)) @@ -525,7 +519,7 @@ int mgs_ir_update(const struct lu_env *env, struct mgs_device *mgs, rc = mgs_nidtbl_write(env, fsdb, mti); if (rc) - return rc; + GOTO(out, rc); /* check ir state */ mutex_lock(&fsdb->fsdb_mutex); @@ -551,7 +545,10 @@ int mgs_ir_update(const struct lu_env *env, struct mgs_device *mgs, atomic_inc(&fsdb->fsdb_notify_phase); wake_up(&fsdb->fsdb_notify_waitq); } - return 0; + +out: + mgs_put_fsdb(mgs, fsdb); + return rc; } /* NID table can be cached by two entities: Clients and MDTs */ @@ -595,7 +592,7 @@ int mgs_get_ir_logs(struct ptlrpc_request *req) { struct lu_env *env = req->rq_svc_thread->t_env; struct mgs_device *mgs = exp2mgs_dev(req->rq_export); - struct fs_db *fsdb; + struct fs_db *fsdb = NULL; struct mgs_config_body *body; struct mgs_config_res *res; struct ptlrpc_bulk_desc *desc; @@ -623,27 +620,27 @@ int mgs_get_ir_logs(struct ptlrpc_request *req) if (rc) RETURN(rc); + bufsize = body->mcb_units << body->mcb_bits; + nrpages = (bufsize + PAGE_SIZE - 1) >> PAGE_SHIFT; + if (nrpages > PTLRPC_MAX_BRW_PAGES) + RETURN(-EINVAL); + rc = mgs_find_or_make_fsdb(env, mgs, fsname, &fsdb); - if (rc) + if (rc) RETURN(rc); - bufsize = body->mcb_units << body->mcb_bits; - nrpages = (bufsize + PAGE_SIZE - 1) >> PAGE_SHIFT; - if (nrpages > PTLRPC_MAX_BRW_PAGES) - RETURN(-EINVAL); - - CDEBUG(D_MGS, "Reading IR log %s bufsize %ld.\n", - body->mcb_name, bufsize); + CDEBUG(D_MGS, "Reading IR log %s bufsize %ld.\n", + body->mcb_name, bufsize); - OBD_ALLOC(pages, sizeof(*pages) * nrpages); - if (pages == NULL) - RETURN(-ENOMEM); + OBD_ALLOC(pages, sizeof(*pages) * nrpages); + if (!pages) + GOTO(out, rc = -ENOMEM); - res = req_capsule_server_get(&req->rq_pill, &RMF_MGS_CONFIG_RES); - if (res == NULL) - GOTO(out, rc = -EINVAL); + res = req_capsule_server_get(&req->rq_pill, &RMF_MGS_CONFIG_RES); + if (!res) + GOTO(out, rc = -EINVAL); - res->mcr_offset = body->mcb_offset; + res->mcr_offset = body->mcb_offset; unit_size = min_t(int, 1 << body->mcb_bits, PAGE_SIZE); bytes = mgs_nidtbl_read(req->rq_export, &fsdb->fsdb_nidtbl, res, pages, nrpages, bufsize / unit_size, unit_size); @@ -658,7 +655,7 @@ int mgs_get_ir_logs(struct ptlrpc_request *req) PTLRPC_BULK_BUF_KIOV, MGS_BULK_PORTAL, &ptlrpc_bulk_kiov_pin_ops); - if (desc == NULL) + if (!desc) GOTO(out, rc = -ENOMEM); for (i = 0; i < page_count && bytes > 0; i++) { @@ -666,18 +663,28 @@ int mgs_get_ir_logs(struct ptlrpc_request *req) min_t(int, bytes, PAGE_SIZE)); bytes -= PAGE_SIZE; - } + } - rc = target_bulk_io(req->rq_export, desc, &lwi); + rc = target_bulk_io(req->rq_export, desc, &lwi); ptlrpc_free_bulk(desc); + GOTO(out, rc); + out: - for (i = 0; i < nrpages; i++) { - if (pages[i] == NULL) - break; - __free_page(pages[i]); + if (pages) { + for (i = 0; i < nrpages; i++) { + if (!pages[i]) + break; + + __free_page(pages[i]); + } + + OBD_FREE(pages, sizeof(*pages) * nrpages); } - OBD_FREE(pages, sizeof(*pages) * nrpages); + + if (fsdb) + mgs_put_fsdb(mgs, fsdb); + return rc; } @@ -840,9 +847,9 @@ ssize_t lprocfs_ir_timeout_seq_write(struct file *file, int mgs_fsc_attach(const struct lu_env *env, struct obd_export *exp, char *fsname) { - struct mgs_export_data *data = &exp->u.eu_mgs_data; + struct mgs_export_data *data = &exp->u.eu_mgs_data; struct mgs_device *mgs = exp2mgs_dev(exp); - struct fs_db *fsdb; + struct fs_db *fsdb = NULL; struct mgs_fsc *fsc = NULL; struct mgs_fsc *new_fsc = NULL; bool found = false; @@ -855,8 +862,8 @@ int mgs_fsc_attach(const struct lu_env *env, struct obd_export *exp, /* allocate a new fsc in case we need it in spinlock. */ OBD_ALLOC_PTR(new_fsc); - if (new_fsc == NULL) - RETURN(-ENOMEM); + if (!new_fsc) + GOTO(out, rc = -ENOMEM); INIT_LIST_HEAD(&new_fsc->mfc_export_list); INIT_LIST_HEAD(&new_fsc->mfc_fsdb_list); @@ -899,7 +906,10 @@ int mgs_fsc_attach(const struct lu_env *env, struct obd_export *exp, class_export_put(new_fsc->mfc_export); OBD_FREE_PTR(new_fsc); } - RETURN(rc); + +out: + mgs_put_fsdb(mgs, fsdb); + RETURN(rc); } void mgs_fsc_cleanup(struct obd_export *exp) -- 1.8.3.1