From 69c6c678e9b67b122d520475b9199e56b62b4dd2 Mon Sep 17 00:00:00 2001 From: tappro Date: Thu, 29 Oct 2009 12:02:07 +0000 Subject: [PATCH] Branch HEAD b=20802 i=adilger i=alex don't sync upon client slot deletion, do server update at first instead. --- lustre/mdt/mdt_recovery.c | 181 ++++++++++++------------------------- lustre/obdfilter/filter.c | 46 ++++------ lustre/obdfilter/filter_internal.h | 2 +- 3 files changed, 77 insertions(+), 152 deletions(-) diff --git a/lustre/mdt/mdt_recovery.c b/lustre/mdt/mdt_recovery.c index bbad09f..449241b 100644 --- a/lustre/mdt/mdt_recovery.c +++ b/lustre/mdt/mdt_recovery.c @@ -49,8 +49,7 @@ #include "mdt_internal.h" static int mdt_server_data_update(const struct lu_env *env, - struct mdt_device *mdt, - int need_sync); + struct mdt_device *mdt); struct lu_buf *mdt_buf(const struct lu_env *env, void *area, ssize_t len) { @@ -149,51 +148,33 @@ static inline int mdt_last_rcvd_header_read(const struct lu_env *env, if (rc == 0) lsd_le_to_cpu(&mti->mti_lsd, &mdt->mdt_lsd); - CDEBUG(D_INFO, "read last_rcvd header rc = %d:\n" - "uuid = %s\n" - "last_transno = "LPU64"\n", - rc, mdt->mdt_lsd.lsd_uuid, - mdt->mdt_lsd.lsd_last_transno); + CDEBUG(D_INFO, "read last_rcvd header rc = %d, uuid = %s, " + "last_transno = "LPU64"\n", rc, mdt->mdt_lsd.lsd_uuid, + mdt->mdt_lsd.lsd_last_transno); return rc; } -static inline int mdt_last_rcvd_header_write(const struct lu_env *env, - struct mdt_device *mdt, - int need_sync) +static int mdt_last_rcvd_header_write(const struct lu_env *env, + struct mdt_device *mdt, + struct thandle *th) { struct mdt_thread_info *mti; - struct thandle *th; int rc; ENTRY; mti = lu_context_key_get(&env->le_ctx, &mdt_thread_key); - if (mti->mti_exp) { - spin_lock(&mti->mti_exp->exp_lock); - mti->mti_exp->exp_need_sync = need_sync; - spin_unlock(&mti->mti_exp->exp_lock); - } - mdt_trans_credit_init(env, mdt, MDT_TXN_LAST_RCVD_WRITE_OP); - th = mdt_trans_start(env, mdt); - if (IS_ERR(th)) - RETURN(PTR_ERR(th)); - mti->mti_off = 0; lsd_cpu_to_le(&mdt->mdt_lsd, &mti->mti_lsd); - if (need_sync && mti->mti_exp) - mdt_trans_add_cb(th, lut_cb_client, mti->mti_exp); - rc = dt_record_write(env, mdt->mdt_last_rcvd, mdt_buf_const(env, &mti->mti_lsd, sizeof(mti->mti_lsd)), &mti->mti_off, th); - mdt_trans_stop(env, mdt, th); - - CDEBUG(D_INFO, "write last_rcvd header rc = %d:\n" - "uuid = %s\nlast_transno = "LPU64"\n", - rc, mdt->mdt_lsd.lsd_uuid, mdt->mdt_lsd.lsd_last_transno); + CDEBUG(D_INFO, "write last_rcvd header rc = %d, uuid = %s, " + "last_transno = "LPU64"\n", rc, mdt->mdt_lsd.lsd_uuid, + mdt->mdt_lsd.lsd_last_transno); RETURN(rc); } @@ -213,25 +194,14 @@ static int mdt_last_rcvd_read(const struct lu_env *env, if (rc == 0) lcd_le_to_cpu(tmp, lcd); - CDEBUG(D_INFO, "read lcd @%d rc = %d:\n" - "uuid = %s\n" - "last_transno = "LPU64"\n" - "last_xid = "LPU64"\n" - "last_result = %u\n" - "last_data = %u\n" - "last_close_transno = "LPU64"\n" - "last_close_xid = "LPU64"\n" - "last_close_result = %u\n", - (int)(*off - sizeof(*tmp)), - rc, - lcd->lcd_uuid, - lcd->lcd_last_transno, - lcd->lcd_last_xid, - lcd->lcd_last_result, - lcd->lcd_last_data, - lcd->lcd_last_close_transno, - lcd->lcd_last_close_xid, - lcd->lcd_last_close_result); + CDEBUG(D_INFO, "read lcd @%d rc = %d, uuid = %s, last_transno = "LPU64 + ", last_xid = "LPU64", last_result = %u, last_data = %u, " + "last_close_transno = "LPU64", last_close_xid = "LPU64", " + "last_close_result = %u\n", (int)(*off - sizeof(*tmp)), + rc, lcd->lcd_uuid, lcd->lcd_last_transno, lcd->lcd_last_xid, + lcd->lcd_last_result, lcd->lcd_last_data, + lcd->lcd_last_close_transno, lcd->lcd_last_close_xid, + lcd->lcd_last_close_result); return rc; } @@ -253,25 +223,14 @@ static int mdt_last_rcvd_write(const struct lu_env *env, rc = dt_record_write(env, mdt->mdt_last_rcvd, mdt_buf_const(env, tmp, sizeof(*tmp)), off, th); - CDEBUG(D_INFO, "write lcd @%d rc = %d:\n" - "uuid = %s\n" - "last_transno = "LPU64"\n" - "last_xid = "LPU64"\n" - "last_result = %u\n" - "last_data = %u\n" - "last_close_transno = "LPU64"\n" - "last_close_xid = "LPU64"\n" - "last_close_result = %u\n", - (int)(*off - sizeof(*tmp)), - rc, - lcd->lcd_uuid, - lcd->lcd_last_transno, - lcd->lcd_last_xid, - lcd->lcd_last_result, - lcd->lcd_last_data, - lcd->lcd_last_close_transno, - lcd->lcd_last_close_xid, - lcd->lcd_last_close_result); + CDEBUG(D_INFO, "write lcd @%d rc = %d, uuid = %s, last_transno = "LPU64 + ", last_xid = "LPU64", last_result = %u, last_data = %u, " + "last_close_transno = "LPU64", last_close_xid = "LPU64" ," + "last_close_result = %u\n", (int)(*off - sizeof(*tmp)), + rc, lcd->lcd_uuid, lcd->lcd_last_transno, lcd->lcd_last_xid, + lcd->lcd_last_result, lcd->lcd_last_data, + lcd->lcd_last_close_transno, lcd->lcd_last_close_xid, + lcd->lcd_last_close_result); return rc; } @@ -515,8 +474,7 @@ static int mdt_server_data_init(const struct lu_env *env, lsd->lsd_mount_count = mdt->mdt_mount_count; /* save it, so mount count and last_transno is current */ - rc = mdt_server_data_update(env, mdt, (mti->mti_exp && - mti->mti_exp->exp_need_sync)); + rc = mdt_server_data_update(env, mdt); if (rc) GOTO(err_client, rc); @@ -529,11 +487,18 @@ out: } static int mdt_server_data_update(const struct lu_env *env, - struct mdt_device *mdt, - int need_sync) + struct mdt_device *mdt) { - int rc = 0; - ENTRY; + struct mdt_thread_info *mti; + struct thandle *th; + int rc; + + mti = lu_context_key_get(&env->le_ctx, &mdt_thread_key); + + mdt_trans_credit_init(env, mdt, MDT_TXN_LAST_RCVD_WRITE_OP); + th = mdt_trans_start(env, mdt); + if (IS_ERR(th)) + RETURN(PTR_ERR(th)); CDEBUG(D_SUPER, "MDS mount_count is "LPU64", last_transno is "LPU64"\n", mdt->mdt_mount_count, mdt->mdt_last_transno); @@ -542,15 +507,12 @@ static int mdt_server_data_update(const struct lu_env *env, mdt->mdt_lsd.lsd_last_transno = mdt->mdt_last_transno; spin_unlock(&mdt->mdt_transno_lock); - /* - * This may be called from difficult reply handler and - * mdt->mdt_last_rcvd may be NULL that time. - */ - if (mdt->mdt_last_rcvd != NULL) - rc = mdt_last_rcvd_header_write(env, mdt, need_sync); - RETURN(rc); + rc = mdt_last_rcvd_header_write(env, mdt, th); + mdt_trans_stop(env, mdt, th); + return rc; } + int mdt_client_new(const struct lu_env *env, struct mdt_device *mdt) { unsigned long *bitmap = mdt->mdt_client_bitmap; @@ -686,7 +648,6 @@ int mdt_client_del(const struct lu_env *env, struct mdt_device *mdt) struct obd_device *obd = mdt2obd_dev(mdt); struct obd_export *exp; struct thandle *th; - int need_sync; loff_t off; int rc = 0; ENTRY; @@ -729,57 +690,27 @@ int mdt_client_del(const struct lu_env *env, struct mdt_device *mdt) LBUG(); } - /* Don't force sync on disconnect if aborting recovery, - * or it does num_clients * num_osts. b=17194 */ - need_sync = (!exp->exp_libclient || exp->exp_need_sync) && - !(exp->exp_flags & OBD_OPT_ABORT_RECOV); - - /* - * This may be called from difficult reply handler path and - * mdt->mdt_last_rcvd may be NULL that time. - */ - if (mdt->mdt_last_rcvd != NULL) { - mdt_trans_credit_init(env, mdt, MDT_TXN_LAST_RCVD_WRITE_OP); - - spin_lock(&exp->exp_lock); - exp->exp_need_sync = need_sync; - spin_unlock(&exp->exp_lock); - - th = mdt_trans_start(env, mdt); - if (IS_ERR(th)) - GOTO(free, rc = PTR_ERR(th)); - - if (need_sync) { - /* - * Until this operations will be committed the sync - * is needed for this export. - */ - mdt_trans_add_cb(th, lut_cb_client, exp); - } + /* write server data at first so last_transno will be save in it in + * any case */ + mdt_server_data_update(env, mdt); - mutex_down(&med->med_lcd_lock); - memset(lcd, 0, sizeof *lcd); - - rc = mdt_last_rcvd_write(env, mdt, lcd, &off, th); - mutex_up(&med->med_lcd_lock); - mdt_trans_stop(env, mdt, th); - } + mdt_trans_credit_init(env, mdt, MDT_TXN_LAST_RCVD_WRITE_OP); + th = mdt_trans_start(env, mdt); + if (IS_ERR(th)) + GOTO(free, rc = PTR_ERR(th)); - CDEBUG(rc == 0 ? D_INFO : D_ERROR, "Zeroing out client idx %u in " - "%s %ssync rc %d\n", med->med_lr_idx, LAST_RCVD, - need_sync ? "" : "a", rc); + mutex_down(&med->med_lcd_lock); + memset(lcd, 0, sizeof *lcd); + rc = mdt_last_rcvd_write(env, mdt, lcd, &off, th); + mutex_up(&med->med_lcd_lock); + mdt_trans_stop(env, mdt, th); spin_lock(&mdt->mdt_client_bitmap_lock); clear_bit(med->med_lr_idx, mdt->mdt_client_bitmap); spin_unlock(&mdt->mdt_client_bitmap_lock); - /* - * Make sure the server's last_transno is up to date. Do this - * after the client is freed so we know all the client's - * transactions have been committed. - */ - mdt_server_data_update(env, mdt, need_sync); - + CDEBUG(rc == 0 ? D_INFO : D_ERROR, "Zeroing out client idx %u in " + "%s, rc %d\n", med->med_lr_idx, LAST_RCVD, rc); EXIT; free: OBD_FREE_PTR(lcd); diff --git a/lustre/obdfilter/filter.c b/lustre/obdfilter/filter.c index 5016618..e159b7b 100644 --- a/lustre/obdfilter/filter.c +++ b/lustre/obdfilter/filter.c @@ -429,28 +429,20 @@ static int filter_client_free(struct obd_export *exp) LBUG(); } - if (!(exp->exp_flags & OBD_OPT_FAILOVER)) { - /* Don't force sync on disconnect if aborting recovery, - * or it does num_clients * num_osts. b=17194 */ - int need_sync = exp->exp_need_sync && - !(exp->exp_flags&OBD_OPT_ABORT_RECOV); - push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - rc = fsfilt_write_record(obd, filter->fo_rcvd_filp, &zero_lcd, - sizeof(zero_lcd), &off, 0); + push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + /* Make sure the server's last_transno is up to date. + * This should be done before zeroing client slot so last_transno will + * be in server data or in client data in case of failure */ + filter_update_server_data(obd, filter->fo_rcvd_filp, filter->fo_fsd); - /* Make sure the server's last_transno is up to date. Do this - * after the client is freed so we know all the client's - * transactions have been committed. */ - if (rc == 0) - filter_update_server_data(obd, filter->fo_rcvd_filp, - filter->fo_fsd, need_sync); - pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); + rc = fsfilt_write_record(obd, filter->fo_rcvd_filp, &zero_lcd, + sizeof(zero_lcd), &off, 0); + pop_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); - CDEBUG(rc == 0 ? D_INFO : D_ERROR, - "zero out client %s at idx %u/%llu in %s %ssync rc %d\n", - fed->fed_lcd->lcd_uuid, fed->fed_lr_idx, fed->fed_lr_off, - LAST_RCVD, need_sync ? "" : "a", rc); - } + CDEBUG(rc == 0 ? D_INFO : D_ERROR, + "zero out client %s at idx %u/%llu in %s, rc %d\n", + fed->fed_lcd->lcd_uuid, fed->fed_lr_idx, fed->fed_lr_off, + LAST_RCVD, rc); if (!test_and_clear_bit(fed->fed_lr_idx, filter->fo_last_rcvd_slots)) { CERROR("FILTER client %u: bit already clear in bitmap!!\n", @@ -660,7 +652,7 @@ static int filter_free_server_data(struct filter_obd *filter) /* assumes caller is already in kernel ctxt */ int filter_update_server_data(struct obd_device *obd, struct file *filp, - struct lr_server_data *fsd, int force_sync) + struct lr_server_data *fsd) { loff_t off = 0; int rc; @@ -672,7 +664,7 @@ int filter_update_server_data(struct obd_device *obd, struct file *filp, CDEBUG(D_INODE, "server last_mount: "LPU64"\n", le64_to_cpu(fsd->lsd_mount_count)); - rc = fsfilt_write_record(obd, filp, fsd, sizeof(*fsd), &off, force_sync); + rc = fsfilt_write_record(obd, filp, fsd, sizeof(*fsd), &off, 0); if (rc) CERROR("error writing lr_server_data: rc = %d\n", rc); @@ -908,7 +900,7 @@ out: fsd->lsd_mount_count = cpu_to_le64(filter->fo_mount_count); /* save it, so mount count and last_transno is current */ - rc = filter_update_server_data(obd, filp, filter->fo_fsd, 1); + rc = filter_update_server_data(obd, filp, filter->fo_fsd); if (rc) GOTO(err_client, rc); @@ -1360,7 +1352,7 @@ static void filter_post(struct obd_device *obd) push_ctxt(&saved, &obd->obd_lvfs_ctxt, NULL); rc = filter_update_server_data(obd, filter->fo_rcvd_filp, - filter->fo_fsd, 0); + filter->fo_fsd); if (rc) CERROR("error writing server data: rc = %d\n", rc); @@ -2679,7 +2671,7 @@ static int filter_connect_internal(struct obd_export *exp, LPU64" left: "LPU64"\n", exp->exp_obd->obd_name, exp->exp_client_uuid.uuid, exp, data->ocd_grant, want, left); - + filter->fo_tot_granted_clients ++; } @@ -2693,8 +2685,10 @@ static int filter_connect_internal(struct obd_export *exp, /* this will only happen on the first connect */ lsd->lsd_ost_index = cpu_to_le32(data->ocd_index); lsd->lsd_feature_compat |= cpu_to_le32(OBD_COMPAT_OST); + /* sync is not needed here as filter_client_add will + * set exp_need_sync flag */ filter_update_server_data(exp->exp_obd, - filter->fo_rcvd_filp, lsd, 1); + filter->fo_rcvd_filp, lsd); } else if (index != data->ocd_index) { LCONSOLE_ERROR_MSG(0x136, "Connection from %s to index" " %u doesn't match actual OST index" diff --git a/lustre/obdfilter/filter_internal.h b/lustre/obdfilter/filter_internal.h index c203f27..748c8b4 100644 --- a/lustre/obdfilter/filter_internal.h +++ b/lustre/obdfilter/filter_internal.h @@ -138,7 +138,7 @@ __u64 filter_last_id(struct filter_obd *, obd_gr group); int filter_update_fidea(struct obd_export *exp, struct inode *inode, void *handle, struct obdo *oa); int filter_update_server_data(struct obd_device *, struct file *, - struct lr_server_data *, int force_sync); + struct lr_server_data *); int filter_update_last_objid(struct obd_device *, obd_gr, int force_sync); int filter_common_setup(struct obd_device *, struct lustre_cfg *lcfg, void *option); -- 1.8.3.1