From: Eric Mei Date: Tue, 18 May 2010 14:59:35 +0000 (-0600) Subject: b=22458 fix concurrent mgs lock revocation. X-Git-Tag: v1_10_0_43~41 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=708bc9a21e4437a163e60492a0813a9bdf85dd29;hp=59f118cf249b240267f26de8a2a5d267a9848920 b=22458 fix concurrent mgs lock revocation. r=nathan r=rread --- diff --git a/lustre/include/lustre_net.h b/lustre/include/lustre_net.h index 9bd4205..946490a 100644 --- a/lustre/include/lustre_net.h +++ b/lustre/include/lustre_net.h @@ -164,7 +164,8 @@ /* SEQ_MAXREPSIZE == lustre_msg + ptlrpc_body + lu_range */ #define SEQ_MAXREPSIZE (152) -#define MGS_THREADS_AUTO_MIN 2 +/* MGS threads must be >= 3, see bug 22458 comment #28 */ +#define MGS_THREADS_AUTO_MIN 3 #define MGS_THREADS_AUTO_MAX 32 #define MGS_NBUFS (64 * cfs_num_online_cpus()) #define MGS_BUFSIZE (8 * 1024) diff --git a/lustre/mgs/mgs_handler.c b/lustre/mgs/mgs_handler.c index 1d687e9..fbbea81 100644 --- a/lustre/mgs/mgs_handler.c +++ b/lustre/mgs/mgs_handler.c @@ -350,18 +350,23 @@ static int mgs_put_cfg_lock(struct lustre_handle *lockh) RETURN(0); } -static void mgs_revoke_lock(struct obd_device *obd, char *fsname, - struct lustre_handle *lockh) +void mgs_revoke_lock(struct obd_device *obd, struct fs_db *fsdb) { - int lockrc; + struct lustre_handle lockh; + int lockrc; + + LASSERT(fsdb->fsdb_name[0] != '\0'); + + if (cfs_test_and_set_bit(1, &fsdb->fsdb_revoking_lock) == 0) { + lockrc = mgs_get_cfg_lock(obd, fsdb->fsdb_name, &lockh); + /* clear the bit before lock put */ + cfs_clear_bit(1, &fsdb->fsdb_revoking_lock); - if (fsname[0]) { - lockrc = mgs_get_cfg_lock(obd, fsname, lockh); if (lockrc != ELDLM_OK) - CERROR("lock error %d for fs %s\n", lockrc, - fsname); + CERROR("lock error %d for fs %s\n", + lockrc, fsdb->fsdb_name); else - mgs_put_cfg_lock(lockh); + mgs_put_cfg_lock(&lockh); } } @@ -400,9 +405,9 @@ static int mgs_check_target(struct obd_device *obd, struct mgs_target_info *mti) static int mgs_handle_target_reg(struct ptlrpc_request *req) { struct obd_device *obd = req->rq_export->exp_obd; - struct lustre_handle lockh; struct mgs_target_info *mti, *rep_mti; - int rc = 0, lockrc; + struct fs_db *fsdb; + int rc = 0; ENTRY; mgs_counter_incr(req->rq_export, LPROC_MGS_TARGET_REG); @@ -420,24 +425,8 @@ static int mgs_handle_target_reg(struct ptlrpc_request *req) GOTO(out_nolock, rc); } - /* Revoke the config lock to make sure nobody is reading. */ - /* Although actually I think it should be alright if - someone was reading while we were updating the logs - if we - revoke at the end they will just update from where they left off. */ - lockrc = mgs_get_cfg_lock(obd, mti->mti_fsname, &lockh); - if (lockrc != ELDLM_OK) { - LCONSOLE_ERROR_MSG(0x13d, "%s: Can't signal other nodes to " - "update their configuration (%d). Updating " - "local logs anyhow; you might have to " - "manually restart other nodes to get the " - "latest configuration.\n", - obd->obd_name, lockrc); - } - OBD_FAIL_TIMEOUT(OBD_FAIL_MGS_PAUSE_TARGET_REG, 10); - /* Log writing contention is handled by the fsdb_sem */ - if (mti->mti_flags & LDD_F_WRITECONF) { if (mti->mti_flags & LDD_F_SV_TYPE_MDT && mti->mti_stripe_index == 0) { @@ -458,9 +447,23 @@ static int mgs_handle_target_reg(struct ptlrpc_request *req) mti->mti_flags &= ~LDD_F_UPGRADE14; } + rc = mgs_find_or_make_fsdb(obd, mti->mti_fsname, &fsdb); + if (rc) { + CERROR("Can't get db for %s: %d\n", mti->mti_fsname, rc); + GOTO(out_nolock, rc); + } + + /* + * Log writing contention is handled by the fsdb_sem. + * + * It should be alright if someone was reading while we were + * updating the logs - if we revoke at the end they will just update + * from where they left off. + */ + /* COMPAT_146 */ if (mti->mti_flags & LDD_F_UPGRADE14) { - rc = mgs_upgrade_sv_14(obd, mti); + rc = mgs_upgrade_sv_14(obd, mti, fsdb); if (rc) { CERROR("Can't upgrade from 1.4 (%d)\n", rc); GOTO(out, rc); @@ -477,7 +480,7 @@ static int mgs_handle_target_reg(struct ptlrpc_request *req) /* create or update the target log and update the client/mdt logs */ - rc = mgs_write_log_target(obd, mti); + rc = mgs_write_log_target(obd, mti, fsdb); if (rc) { CERROR("Failed to write %s log (%d)\n", mti->mti_svname, rc); @@ -491,9 +494,8 @@ static int mgs_handle_target_reg(struct ptlrpc_request *req) } out: - /* done with log update */ - if (lockrc == ELDLM_OK) - mgs_put_cfg_lock(&lockh); + mgs_revoke_lock(obd, fsdb); + out_nolock: CDEBUG(D_MGS, "replying with %s, index=%d, rc=%d\n", mti->mti_svname, mti->mti_stripe_index, rc); @@ -514,7 +516,6 @@ static int mgs_set_info_rpc(struct ptlrpc_request *req) { struct obd_device *obd = req->rq_export->exp_obd; struct mgs_send_param *msp, *rep_msp; - struct lustre_handle lockh; int rc; struct lustre_cfg_bufs bufs; struct lustre_cfg *lcfg; @@ -535,9 +536,6 @@ static int mgs_set_info_rpc(struct ptlrpc_request *req) RETURN(rc); } - /* request for update */ - mgs_revoke_lock(obd, fsname, &lockh); - lustre_cfg_free(lcfg); rc = req_capsule_server_pack(&req->rq_pill); @@ -796,7 +794,6 @@ static int mgs_iocontrol_pool(struct obd_device *obd, struct obd_ioctl_data *data) { int rc; - struct lustre_handle lockh; struct lustre_cfg *lcfg = NULL; struct llog_rec_hdr rec; char *fsname = NULL; @@ -883,9 +880,6 @@ static int mgs_iocontrol_pool(struct obd_device *obd, GOTO(out_pool, rc); } - /* request for update */ - mgs_revoke_lock(obd, fsname, &lockh); - out_pool: if (lcfg != NULL) OBD_FREE(lcfg, data->ioc_plen1); @@ -914,7 +908,6 @@ int mgs_iocontrol(unsigned int cmd, struct obd_export *exp, int len, switch (cmd) { case OBD_IOC_PARAM: { - struct lustre_handle lockh; struct lustre_cfg *lcfg; struct llog_rec_hdr rec; char fsname[MTI_NAME_MAXLEN]; @@ -942,13 +935,6 @@ int mgs_iocontrol(unsigned int cmd, struct obd_export *exp, int len, CERROR("setparam err %d\n", rc); GOTO(out_free, rc); } - - /* Revoke lock so everyone updates. Should be alright if - someone was already reading while we were updating the logs, - so we don't really need to hold the lock while we're - writing (above). */ - mgs_revoke_lock(obd, fsname, &lockh); - out_free: OBD_FREE(lcfg, data->ioc_plen1); RETURN(rc); diff --git a/lustre/mgs/mgs_internal.h b/lustre/mgs/mgs_internal.h index f62cb29..4c77ed3 100644 --- a/lustre/mgs/mgs_internal.h +++ b/lustre/mgs/mgs_internal.h @@ -81,6 +81,8 @@ struct fs_db { __u32 fsdb_flags; __u32 fsdb_gen; + __u8 fsdb_revoking_lock; /* lock is being revoked */ + /* in-memory copy of the srpc rules, guarded by fsdb_sem */ struct sptlrpc_rule_set fsdb_srpc_gen; struct mgs_tgt_srpc_conf *fsdb_srpc_tgt; @@ -96,8 +98,10 @@ int mgs_find_or_make_fsdb(struct obd_device *obd, char *name, int mgs_get_fsdb_srpc_from_llog(struct obd_device *obd, struct fs_db *fsdb); int mgs_check_index(struct obd_device *obd, struct mgs_target_info *mti); int mgs_check_failnid(struct obd_device *obd, struct mgs_target_info *mti); -int mgs_write_log_target(struct obd_device *obd, struct mgs_target_info *mti); -int mgs_upgrade_sv_14(struct obd_device *obd, struct mgs_target_info *mti); +int mgs_write_log_target(struct obd_device *obd, struct mgs_target_info *mti, + struct fs_db *fsdb); +int mgs_upgrade_sv_14(struct obd_device *obd, struct mgs_target_info *mti, + struct fs_db *fsdb); int mgs_erase_log(struct obd_device *obd, char *name); int mgs_erase_logs(struct obd_device *obd, char *fsname); int mgs_setparam(struct obd_device *obd, struct lustre_cfg *lcfg, char *fsname); @@ -105,6 +109,9 @@ int mgs_setparam(struct obd_device *obd, struct lustre_cfg *lcfg, char *fsname); int mgs_pool_cmd(struct obd_device *obd, enum lcfg_command_type cmd, char *poolname, char *fsname, char *ostname); +/* mgs_handler.c */ +void mgs_revoke_lock(struct obd_device *obd, struct fs_db *fsdb); + /* mgs_fs.c */ int mgs_client_add(struct obd_device *obd, struct obd_export *exp, void *localdata); diff --git a/lustre/mgs/mgs_llog.c b/lustre/mgs/mgs_llog.c index 46b9569..5cb8264 100644 --- a/lustre/mgs/mgs_llog.c +++ b/lustre/mgs/mgs_llog.c @@ -2649,9 +2649,9 @@ int mgs_check_failnid(struct obd_device *obd, struct mgs_target_info *mti) } int mgs_write_log_target(struct obd_device *obd, - struct mgs_target_info *mti) + struct mgs_target_info *mti, + struct fs_db *fsdb) { - struct fs_db *fsdb; int rc = -EINVAL; char *buf, *params; ENTRY; @@ -2696,12 +2696,6 @@ int mgs_write_log_target(struct obd_device *obd, } } - rc = mgs_find_or_make_fsdb(obd, mti->mti_fsname, &fsdb); - if (rc) { - CERROR("Can't get db for %s\n", mti->mti_fsname); - RETURN(rc); - } - cfs_down(&fsdb->fsdb_sem); if (mti->mti_flags & @@ -2757,9 +2751,9 @@ out_up: /* COMPAT_146 */ /* verify that we can handle the old config logs */ -int mgs_upgrade_sv_14(struct obd_device *obd, struct mgs_target_info *mti) +int mgs_upgrade_sv_14(struct obd_device *obd, struct mgs_target_info *mti, + struct fs_db *fsdb) { - struct fs_db *fsdb; int rc = 0; ENTRY; @@ -2778,10 +2772,6 @@ int mgs_upgrade_sv_14(struct obd_device *obd, struct mgs_target_info *mti) LCONSOLE_INFO("upgrading server %s from pre-1.6\n", mti->mti_svname); server_mti_print("upgrade", mti); - rc = mgs_find_or_make_fsdb(obd, mti->mti_fsname, &fsdb); - if (rc) - RETURN(rc); - if (fsdb->fsdb_flags & FSDB_LOG_EMPTY) { LCONSOLE_ERROR_MSG(0x14a, "The old client log %s-client is " "missing. Was tunefs.lustre successful?\n", @@ -3004,6 +2994,13 @@ int mgs_setparam(struct obd_device *obd, struct lustre_cfg *lcfg, char *fsname) rc = mgs_write_log_param(obd, fsdb, mti, mti->mti_params); cfs_up(&fsdb->fsdb_sem); + /* + * Revoke lock so everyone updates. Should be alright if + * someone was already reading while we were updating the logs, + * so we don't really need to hold the lock while we're + * writing (above). + */ + mgs_revoke_lock(obd, fsdb); out: OBD_FREE_PTR(mti); RETURN(rc); @@ -3142,6 +3139,8 @@ int mgs_pool_cmd(struct obd_device *obd, enum lcfg_command_type cmd, name_destroy(&logname); cfs_up(&fsdb->fsdb_sem); + /* request for update */ + mgs_revoke_lock(obd, fsdb); EXIT; out: