X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lustre%2Ftarget%2Ftgt_lastrcvd.c;h=ea70037a3cee6b943f6319784750e723cafd6184;hp=39b697ce78c65754214fc3bd9e4967e09b8f17e7;hb=11eda8742fa4505bc62c6113b067a75687f171cc;hpb=0754bc8f2623bea184111af216f7567608db35b6 diff --git a/lustre/target/tgt_lastrcvd.c b/lustre/target/tgt_lastrcvd.c index 39b697c..ea70037 100644 --- a/lustre/target/tgt_lastrcvd.c +++ b/lustre/target/tgt_lastrcvd.c @@ -15,11 +15,7 @@ * * You should have received a copy of the GNU General Public License * version 2 along with this program; If not, see - * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf - * - * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, - * CA 95054 USA or visit www.sun.com if you need additional information or - * have any questions. + * http://www.gnu.org/licenses/gpl-2.0.html * * GPL HEADER END */ @@ -27,7 +23,7 @@ * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. * - * Copyright (c) 2011, 2015, Intel Corporation. + * Copyright (c) 2011, 2016, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -50,7 +46,8 @@ static int tgt_bitmap_chunk_alloc(struct lu_target *lut, int chunk) { unsigned long *bm; - OBD_ALLOC(bm, BITS_TO_LONGS(LUT_REPLY_SLOTS_PER_CHUNK) * sizeof(long)); + OBD_ALLOC_LARGE(bm, BITS_TO_LONGS(LUT_REPLY_SLOTS_PER_CHUNK) * + sizeof(long)); if (bm == NULL) return -ENOMEM; @@ -59,7 +56,7 @@ static int tgt_bitmap_chunk_alloc(struct lu_target *lut, int chunk) if (lut->lut_reply_bitmap[chunk] != NULL) { /* someone else already allocated the bitmap for this chunk */ spin_unlock(&lut->lut_client_bitmap_lock); - OBD_FREE(bm, BITS_TO_LONGS(LUT_REPLY_SLOTS_PER_CHUNK) * + OBD_FREE_LARGE(bm, BITS_TO_LONGS(LUT_REPLY_SLOTS_PER_CHUNK) * sizeof(long)); return 0; } @@ -155,6 +152,12 @@ static int tgt_clear_reply_slot(struct lu_target *lut, int idx) LASSERT(chunk < LUT_REPLY_SLOTS_MAX_CHUNKS); LASSERT(b < LUT_REPLY_SLOTS_PER_CHUNK); + if (lut->lut_reply_bitmap[chunk] == NULL) { + CERROR("%s: slot %d not allocated\n", + tgt_name(lut), idx); + return -ENOENT; + } + if (test_and_clear_bit(b, lut->lut_reply_bitmap[chunk]) == 0) { CERROR("%s: slot %d already clear in bitmap\n", tgt_name(lut), idx); @@ -308,7 +311,7 @@ static void tgt_free_reply_data(struct lu_target *lut, { CDEBUG(D_TRACE, "%s: free reply data %p: xid %llu, transno %llu, " "client gen %u, slot idx %d\n", - tgt_name(lut), trd, trd->trd_reply.lrd_xid, + lut == NULL ? "" : tgt_name(lut), trd, trd->trd_reply.lrd_xid, trd->trd_reply.lrd_transno, trd->trd_reply.lrd_client_gen, trd->trd_index); @@ -316,7 +319,8 @@ static void tgt_free_reply_data(struct lu_target *lut, list_del(&trd->trd_list); ted->ted_reply_cnt--; - tgt_clear_reply_slot(lut, trd->trd_index); + if (lut != NULL) + tgt_clear_reply_slot(lut, trd->trd_index); OBD_FREE_PTR(trd); } @@ -331,7 +335,7 @@ static void tgt_release_reply_data(struct lu_target *lut, { CDEBUG(D_TRACE, "%s: release reply data %p: xid %llu, transno %llu, " "client gen %u, slot idx %d\n", - tgt_name(lut), trd, trd->trd_reply.lrd_xid, + lut == NULL ? "" : tgt_name(lut), trd, trd->trd_reply.lrd_xid, trd->trd_reply.lrd_transno, trd->trd_reply.lrd_client_gen, trd->trd_index); @@ -377,6 +381,9 @@ int tgt_client_alloc(struct obd_export *exp) ENTRY; LASSERT(exp != exp->exp_obd->obd_self_export); + spin_lock_init(&exp->exp_target_data.ted_nodemap_lock); + INIT_LIST_HEAD(&exp->exp_target_data.ted_nodemap_member); + OBD_ALLOC_PTR(exp->exp_target_data.ted_lcd); if (exp->exp_target_data.ted_lcd == NULL) RETURN(-ENOMEM); @@ -418,9 +425,12 @@ void tgt_client_free(struct obd_export *exp) OBD_FREE_PTR(ted->ted_lcd); ted->ted_lcd = NULL; - /* Slot may be not yet assigned */ - if (ted->ted_lr_idx < 0) + /* Target may have been freed (see LU-7430) + * Slot may be not yet assigned */ + if (exp->exp_obd->u.obt.obt_magic != OBT_MAGIC || + ted->ted_lr_idx < 0) return; + /* Clear bit when lcd is freed */ LASSERT(lut && lut->lut_client_bitmap); if (!test_and_clear_bit(ted->ted_lr_idx, lut->lut_client_bitmap)) { @@ -450,9 +460,9 @@ int tgt_client_data_read(const struct lu_env *env, struct lu_target *tgt, ptlrpc_status_ntoh(lcd->lcd_last_close_result); } - CDEBUG(D_INFO, "%s: read lcd @%lld uuid = %s, last_transno = "LPU64 - ", last_xid = "LPU64", last_result = %u, last_data = %u, " - "last_close_transno = "LPU64", last_close_xid = "LPU64", " + CDEBUG(D_INFO, "%s: read lcd @%lld uuid = %s, last_transno = %llu" + ", last_xid = %llu, last_result = %u, last_data = %u, " + "last_close_transno = %llu, last_close_xid = %llu, " "last_close_result = %u, rc = %d\n", tgt->lut_obd->obd_name, *off, lcd->lcd_uuid, lcd->lcd_last_transno, lcd->lcd_last_xid, lcd->lcd_last_result, lcd->lcd_last_data, @@ -512,6 +522,9 @@ static int tgt_client_data_update(const struct lu_env *env, rc = dt_trans_start_local(env, tgt->lut_bottom, th); if (rc) GOTO(out, rc); + + mutex_lock(&ted->ted_lcd_lock); + /* * Until this operations will be committed the sync is needed * for this export. This should be done _after_ starting the @@ -530,11 +543,14 @@ static int tgt_client_data_update(const struct lu_env *env, tti->tti_off = ted->ted_lr_off; rc = tgt_client_data_write(env, tgt, ted->ted_lcd, &tti->tti_off, th); + + mutex_unlock(&ted->ted_lcd_lock); + EXIT; out: dt_trans_stop(env, tgt->lut_bottom, th); CDEBUG(D_INFO, "%s: update last_rcvd client data for UUID = %s, " - "last_transno = "LPU64": rc = %d\n", tgt->lut_obd->obd_name, + "last_transno = %llu: rc = %d\n", tgt->lut_obd->obd_name, tgt->lut_lsd.lsd_uuid, tgt->lut_lsd.lsd_last_transno, rc); return rc; @@ -553,7 +569,7 @@ int tgt_server_data_read(const struct lu_env *env, struct lu_target *tgt) lsd_le_to_cpu(&tti->tti_lsd, &tgt->lut_lsd); CDEBUG(D_INFO, "%s: read last_rcvd server data for UUID = %s, " - "last_transno = "LPU64": rc = %d\n", tgt->lut_obd->obd_name, + "last_transno = %llu: rc = %d\n", tgt->lut_obd->obd_name, tgt->lut_lsd.lsd_uuid, tgt->lut_lsd.lsd_last_transno, rc); return rc; } @@ -575,7 +591,7 @@ int tgt_server_data_write(const struct lu_env *env, struct lu_target *tgt, rc = dt_record_write(env, dto, &tti->tti_buf, &tti->tti_off, th); CDEBUG(D_INFO, "%s: write last_rcvd server data for UUID = %s, " - "last_transno = "LPU64": rc = %d\n", tgt->lut_obd->obd_name, + "last_transno = %llu: rc = %d\n", tgt->lut_obd->obd_name, tgt->lut_lsd.lsd_uuid, tgt->lut_lsd.lsd_last_transno, rc); RETURN(rc); @@ -594,7 +610,7 @@ int tgt_server_data_update(const struct lu_env *env, struct lu_target *tgt, ENTRY; CDEBUG(D_SUPER, - "%s: mount_count is "LPU64", last_transno is "LPU64"\n", + "%s: mount_count is %llu, last_transno is %llu\n", tgt->lut_lsd.lsd_uuid, tgt->lut_obd->u.obt.obt_mount_count, tgt->lut_last_transno); @@ -624,7 +640,7 @@ out: dt_trans_stop(env, tgt->lut_bottom, th); CDEBUG(D_INFO, "%s: update last_rcvd server data for UUID = %s, " - "last_transno = "LPU64": rc = %d\n", tgt->lut_obd->obd_name, + "last_transno = %llu: rc = %d\n", tgt->lut_obd->obd_name, tgt->lut_lsd.lsd_uuid, tgt->lut_lsd.lsd_last_transno, rc); RETURN(rc); } @@ -745,7 +761,7 @@ void tgt_boot_epoch_update(struct lu_target *tgt) } /** - * commit callback, need to update last_commited value + * commit callback, need to update last_committed value */ struct tgt_last_committed_callback { struct dt_txn_commit_cb llcc_cb; @@ -765,6 +781,11 @@ static void tgt_cb_last_committed(struct lu_env *env, struct thandle *th, LASSERT(ccb->llcc_tgt != NULL); LASSERT(ccb->llcc_exp->exp_obd == ccb->llcc_tgt->lut_obd); + /* error hit, don't update last committed to provide chance to + * replay data after fail */ + if (err != 0) + goto out; + /* Fast path w/o spinlock, if exp_last_committed was updated * with higher transno, no need to take spinlock and check, * also no need to update obd_last_committed. */ @@ -777,18 +798,25 @@ static void tgt_cb_last_committed(struct lu_env *env, struct thandle *th, if (ccb->llcc_transno > ccb->llcc_exp->exp_last_committed) { ccb->llcc_exp->exp_last_committed = ccb->llcc_transno; spin_unlock(&ccb->llcc_tgt->lut_translock); + ptlrpc_commit_replies(ccb->llcc_exp); + tgt_cancel_slc_locks(ccb->llcc_tgt, ccb->llcc_transno); } else { spin_unlock(&ccb->llcc_tgt->lut_translock); } + + CDEBUG(D_HA, "%s: transno %lld is committed\n", + ccb->llcc_tgt->lut_obd->obd_name, ccb->llcc_transno); + out: class_export_cb_put(ccb->llcc_exp); - if (ccb->llcc_transno) - CDEBUG(D_HA, "%s: transno "LPD64" is committed\n", - ccb->llcc_tgt->lut_obd->obd_name, ccb->llcc_transno); OBD_FREE_PTR(ccb); } +/** + * Add commit callback function, it returns a non-zero value to inform + * caller to use sync transaction if necessary. + */ int tgt_last_commit_cb_add(struct thandle *th, struct lu_target *tgt, struct obd_export *exp, __u64 transno) { @@ -819,7 +847,9 @@ int tgt_last_commit_cb_add(struct thandle *th, struct lu_target *tgt, /* report failure to force synchronous operation */ return -EPERM; - return rc; + /* if exp_need_sync is set, return non-zero value to force + * a sync transaction. */ + return rc ? rc : exp->exp_need_sync; } struct tgt_new_client_callback { @@ -1026,6 +1056,14 @@ int tgt_client_del(const struct lu_env *env, struct obd_export *exp) exp_connect_flags(exp) & OBD_CONNECT_LIGHTWEIGHT) RETURN(0); + /* Slot may be not yet assigned, use case is race between Client + * reconnect and forced eviction */ + if (ted->ted_lr_idx < 0) { + CWARN("%s: client with UUID '%s' not in bitmap\n", + tgt->lut_obd->obd_name, ted->ted_lcd->lcd_uuid); + RETURN(0); + } + CDEBUG(D_INFO, "%s: del client at idx %u, off %lld, UUID '%s'\n", tgt->lut_obd->obd_name, ted->ted_lr_idx, ted->ted_lr_off, ted->ted_lcd->lcd_uuid); @@ -1042,6 +1080,9 @@ int tgt_client_del(const struct lu_env *env, struct obd_export *exp) if (exp->exp_flags & OBD_OPT_FAILOVER) RETURN(0); + if (OBD_FAIL_CHECK(OBD_FAIL_TGT_CLIENT_DEL)) + RETURN(0); + /* Make sure the server's last_transno is up to date. * This should be done before zeroing client slot so last_transno will * be in server data or in client data in case of failure */ @@ -1053,10 +1094,8 @@ int tgt_client_del(const struct lu_env *env, struct obd_export *exp) RETURN(rc); } - mutex_lock(&ted->ted_lcd_lock); memset(ted->ted_lcd->lcd_uuid, 0, sizeof ted->ted_lcd->lcd_uuid); rc = tgt_client_data_update(env, exp); - mutex_unlock(&ted->ted_lcd_lock); CDEBUG(rc == 0 ? D_INFO : D_ERROR, "%s: zeroing out client %s at idx %u (%llu), rc %d\n", @@ -1154,7 +1193,7 @@ static int tgt_last_rcvd_update(const struct lu_env *env, struct lu_target *tgt, spin_lock(&tgt->lut_translock); if (th->th_result != 0) { if (tti->tti_transno != 0) { - CERROR("%s: replay transno "LPU64" failed: rc = %d\n", + CERROR("%s: replay transno %llu failed: rc = %d\n", tgt_name(tgt), tti->tti_transno, th->th_result); } } else if (tti->tti_transno == 0) { @@ -1173,7 +1212,7 @@ static int tgt_last_rcvd_update(const struct lu_env *env, struct lu_target *tgt, } /* filling reply data */ - CDEBUG(D_INODE, "transno = "LPU64", last_committed = "LPU64"\n", + CDEBUG(D_INODE, "transno = %llu, last_committed = %llu\n", tti->tti_transno, tgt->lut_obd->obd_last_committed); if (req != NULL) { @@ -1286,7 +1325,7 @@ static int tgt_last_rcvd_update(const struct lu_env *env, struct lu_target *tgt, if (*transno_p > tti->tti_transno) { if (!tgt->lut_no_reconstruct) { CERROR("%s: trying to overwrite bigger transno:" - "on-disk: "LPU64", new: "LPU64" replay: " + "on-disk: %llu, new: %llu replay: " "%d. See LU-617.\n", tgt_name(tgt), *transno_p, tti->tti_transno, req_is_replay(req)); @@ -1305,7 +1344,11 @@ static int tgt_last_rcvd_update(const struct lu_env *env, struct lu_target *tgt, if (!lw_client) { tti->tti_off = ted->ted_lr_off; - rc = tgt_client_data_write(env, tgt, ted->ted_lcd, &tti->tti_off, th); + if (CFS_FAIL_CHECK(OBD_FAIL_TGT_RCVD_EIO)) + rc = -EIO; + else + rc = tgt_client_data_write(env, tgt, ted->ted_lcd, + &tti->tti_off, th); if (rc < 0) { mutex_unlock(&ted->ted_lcd_lock); RETURN(rc); @@ -1415,8 +1458,8 @@ static int tgt_clients_data_init(const struct lu_env *env, /* These exports are cleaned up by disconnect, so they * need to be set up like real exports as connect does. */ - CDEBUG(D_HA, "RCVRNG CLIENT uuid: %s idx: %d lr: "LPU64 - " srv lr: "LPU64" lx: "LPU64" gen %u\n", lcd->lcd_uuid, + CDEBUG(D_HA, "RCVRNG CLIENT uuid: %s idx: %d lr: %llu" + " srv lr: %llu lx: %llu gen %u\n", lcd->lcd_uuid, cl_idx, last_transno, lsd->lsd_last_transno, lcd_last_xid(lcd), lcd->lcd_generation); @@ -1470,7 +1513,7 @@ static int tgt_clients_data_init(const struct lu_env *env, } /* Need to check last_rcvd even for duplicated exports. */ - CDEBUG(D_OTHER, "client at idx %d has last_transno = "LPU64"\n", + CDEBUG(D_OTHER, "client at idx %d has last_transno = %llu\n", cl_idx, last_transno); spin_lock(&tgt->lut_translock); @@ -1633,9 +1676,9 @@ int tgt_server_data_init(const struct lu_env *env, struct lu_target *tgt) lsd->lsd_mount_count++; CDEBUG(D_INODE, "=======,=BEGIN DUMPING LAST_RCVD========\n"); - CDEBUG(D_INODE, "%s: server last_transno: "LPU64"\n", + CDEBUG(D_INODE, "%s: server last_transno: %llu\n", tgt_name(tgt), tgt->lut_last_transno); - CDEBUG(D_INODE, "%s: server mount_count: "LPU64"\n", + CDEBUG(D_INODE, "%s: server mount_count: %llu\n", tgt_name(tgt), lsd->lsd_mount_count); CDEBUG(D_INODE, "%s: server data size: %u\n", tgt_name(tgt), lsd->lsd_server_size); @@ -1708,19 +1751,29 @@ int tgt_txn_start_cb(const struct lu_env *env, struct thandle *th, if (tsi->tsi_exp == NULL) return 0; - dto = dt_object_locate(tgt->lut_last_rcvd, th->th_dev); - tti_buf_lcd(tti); - - rc = dt_declare_record_write(env, dto, &tti->tti_buf, - tsi->tsi_exp->exp_target_data.ted_lr_off, - th); - if (rc) - return rc; - - tti_buf_lsd(tti); - rc = dt_declare_record_write(env, dto, &tti->tti_buf, 0, th); - if (rc) - return rc; + if (tgt_is_multimodrpcs_client(tsi->tsi_exp)) { + /* + * Use maximum possible file offset for declaration to ensure + * ZFS will reserve enough credits for a write anywhere in this + * file, since we don't know where in the file the write will be + * because a replay slot has not been assigned. This should be + * replaced by dmu_tx_hold_append() when available. + */ + tti->tti_buf.lb_buf = NULL; + tti->tti_buf.lb_len = sizeof(struct lsd_reply_data); + dto = dt_object_locate(tgt->lut_reply_data, th->th_dev); + rc = dt_declare_record_write(env, dto, &tti->tti_buf, -1, th); + if (rc) + return rc; + } else { + dto = dt_object_locate(tgt->lut_last_rcvd, th->th_dev); + tti_buf_lcd(tti); + tti->tti_off = tsi->tsi_exp->exp_target_data.ted_lr_off; + rc = dt_declare_record_write(env, dto, &tti->tti_buf, + tti->tti_off, th); + if (rc) + return rc; + } if (tsi->tsi_vbr_obj != NULL && !lu_object_remote(&tsi->tsi_vbr_obj->do_lu)) { @@ -1754,7 +1807,7 @@ int tgt_txn_stop_cb(const struct lu_env *env, struct thandle *th, if (tti->tti_has_trans && !echo_client) { if (tti->tti_mult_trans == 0) { - CDEBUG(D_HA, "More than one transaction "LPU64"\n", + CDEBUG(D_HA, "More than one transaction %llu\n", tti->tti_transno); RETURN(0); } @@ -1863,7 +1916,11 @@ int tgt_reply_data_init(const struct lu_env *env, struct lu_target *tgt) /* create in-memory reply_data and link it to * target export's reply list */ - tgt_set_reply_slot(tgt, idx); + rc = tgt_set_reply_slot(tgt, idx); + if (rc != 0) { + mutex_unlock(&ted->ted_lcd_lock); + GOTO(out, rc); + } trd->trd_reply = *lrd; trd->trd_pre_versions[0] = 0; trd->trd_pre_versions[1] = 0;