From 6c3c5c7b6c8f6ecde5fc878f857b0aa0b4fd694f Mon Sep 17 00:00:00 2001 From: Hongchao Zhang Date: Mon, 8 Apr 2013 10:49:09 +0800 Subject: [PATCH] LU-631 osc: wait OSC to complete initial connection with OST in functions lov_prep_*_set, if one OSC isn't active and it is trying its first connection with OST, then spend some time (obd_timeout) to wait it to complete. Change-Id: I424dbf81b6ceebf2cfd1cf48b0f89be40c4c3df4 Signed-off-by: Hongchao Zhang Reviewed-on: http://review.whamcloud.com/2469 Tested-by: Hudson Reviewed-by: Fan Yong Tested-by: Maloo Reviewed-by: Jeremy Filizetti Reviewed-by: Andreas Dilger --- lustre/include/lustre_import.h | 52 +++++++++---- lustre/lov/lov_internal.h | 1 + lustre/lov/lov_request.c | 173 ++++++++++++++++++++++++++--------------- lustre/ptlrpc/import.c | 26 ++++--- 4 files changed, 162 insertions(+), 90 deletions(-) diff --git a/lustre/include/lustre_import.h b/lustre/include/lustre_import.h index cb1c04b..98c2c92 100644 --- a/lustre/include/lustre_import.h +++ b/lustre/include/lustre_import.h @@ -250,23 +250,41 @@ struct obd_import { /** Protects flags, level, generation, conn_cnt, *_list */ spinlock_t imp_lock; - /* flags */ - unsigned long imp_no_timeout:1, /* timeouts are disabled */ - imp_invalid:1, /* evicted */ - imp_deactive:1, /* administratively disabled */ - imp_replayable:1, /* try to recover the import */ - imp_dlm_fake:1, /* don't run recovery (timeout instead) */ - imp_server_timeout:1, /* use 1/2 timeout on MDS' OSCs */ - imp_delayed_recovery:1, /* VBR: imp in delayed recovery */ - imp_no_lock_replay:1, /* VBR: if gap was found then no lock replays */ - imp_vbr_failed:1, /* recovery by versions was failed */ - imp_force_verify:1, /* force an immidiate ping */ - imp_force_next_verify:1,/* force a scheduled ping */ - imp_pingable:1, /* pingable */ - imp_resend_replay:1, /* resend for replay */ - imp_no_pinger_recover:1,/* disable normal recovery, for test only. */ - imp_need_mne_swab:1, /* need IR MNE swab */ - imp_force_reconnect:1; /* import must be reconnected instead of chouse new connection */ + /* flags */ + unsigned long imp_no_timeout:1, /* timeouts are disabled */ + imp_invalid:1, /* evicted */ + /* administratively disabled */ + imp_deactive:1, + /* try to recover the import */ + imp_replayable:1, + /* don't run recovery (timeout instead) */ + imp_dlm_fake:1, + /* use 1/2 timeout on MDS' OSCs */ + imp_server_timeout:1, + /* VBR: imp in delayed recovery */ + imp_delayed_recovery:1, + /* VBR: if gap was found then no lock replays + */ + imp_no_lock_replay:1, + /* recovery by versions was failed */ + imp_vbr_failed:1, + /* force an immidiate ping */ + imp_force_verify:1, + /* force a scheduled ping */ + imp_force_next_verify:1, + /* pingable */ + imp_pingable:1, + /* resend for replay */ + imp_resend_replay:1, + /* disable normal recovery, for test only. */ + imp_no_pinger_recover:1, + /* need IR MNE swab */ + imp_need_mne_swab:1, + /* import must be reconnected instead of + * chouse new connection */ + imp_force_reconnect:1, + /* import has tried to connect with server */ + imp_connect_tried:1; __u32 imp_connect_op; struct obd_connect_data imp_connect_data; __u64 imp_connect_flags_orig; diff --git a/lustre/lov/lov_internal.h b/lustre/lov/lov_internal.h index 7332696..de959ae 100644 --- a/lustre/lov/lov_internal.h +++ b/lustre/lov/lov_internal.h @@ -174,6 +174,7 @@ void lov_update_set(struct lov_request_set *set, struct lov_request *req, int rc); int lov_update_common_set(struct lov_request_set *set, struct lov_request *req, int rc); +int lov_check_and_wait_active(struct lov_obd *lov, int ost_idx); int lov_prep_create_set(struct obd_export *exp, struct obd_info *oifo, struct lov_stripe_md **ea, struct obdo *src_oa, struct obd_trans_info *oti, diff --git a/lustre/lov/lov_request.c b/lustre/lov/lov_request.c index c6d8e84..e6c5181 100644 --- a/lustre/lov/lov_request.c +++ b/lustre/lov/lov_request.c @@ -146,6 +146,62 @@ void lov_set_add_req(struct lov_request *req, struct lov_request_set *set) req->rq_rqset = set; } +static int lov_check_set(struct lov_obd *lov, int idx) +{ + int rc = 0; + mutex_lock(&lov->lov_lock); + + if (lov->lov_tgts[idx] == NULL || + lov->lov_tgts[idx]->ltd_active || + (lov->lov_tgts[idx]->ltd_exp != NULL && + class_exp2cliimp(lov->lov_tgts[idx]->ltd_exp)->imp_connect_tried)) + rc = 1; + + mutex_unlock(&lov->lov_lock); + return rc; +} + +/* Check if the OSC connection exists and is active. + * If the OSC has not yet had a chance to connect to the OST the first time, + * wait once for it to connect instead of returning an error. + */ +int lov_check_and_wait_active(struct lov_obd *lov, int ost_idx) +{ + cfs_waitq_t waitq; + struct l_wait_info lwi; + struct lov_tgt_desc *tgt; + int rc = 0; + + mutex_lock(&lov->lov_lock); + + tgt = lov->lov_tgts[ost_idx]; + + if (unlikely(tgt == NULL)) + GOTO(out, rc = 0); + + if (likely(tgt->ltd_active)) + GOTO(out, rc = 1); + + if (tgt->ltd_exp && class_exp2cliimp(tgt->ltd_exp)->imp_connect_tried) + GOTO(out, rc = 0); + + mutex_unlock(&lov->lov_lock); + + cfs_waitq_init(&waitq); + lwi = LWI_TIMEOUT_INTERVAL(cfs_time_seconds(obd_timeout), + cfs_time_seconds(1), NULL, NULL); + + rc = l_wait_event(waitq, lov_check_set(lov, ost_idx), &lwi); + if (tgt != NULL && tgt->ltd_active) + return 1; + + return 0; + +out: + mutex_unlock(&lov->lov_lock); + return rc; +} + extern void osc_update_enqueue(struct lustre_handle *lov_lockhp, struct lov_oinfo *loi, int flags, struct ost_lvb *lvb, __u32 mode, int rc); @@ -344,11 +400,10 @@ int lov_prep_enqueue_set(struct obd_export *exp, struct obd_info *oinfo, &start, &end)) continue; - if (!lov->lov_tgts[loi->loi_ost_idx] || - !lov->lov_tgts[loi->loi_ost_idx]->ltd_active) { - CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx); - continue; - } + if (!lov_check_and_wait_active(lov, loi->loi_ost_idx)) { + CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx); + continue; + } OBD_ALLOC(req, sizeof(*req)); if (req == NULL) @@ -452,11 +507,10 @@ int lov_prep_match_set(struct obd_export *exp, struct obd_info *oinfo, continue; /* FIXME raid1 should grace this error */ - if (!lov->lov_tgts[loi->loi_ost_idx] || - !lov->lov_tgts[loi->loi_ost_idx]->ltd_active) { - CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx); - GOTO(out_set, rc = -EIO); - } + if (!lov_check_and_wait_active(lov, loi->loi_ost_idx)) { + CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx); + GOTO(out_set, rc = -EIO); + } OBD_ALLOC(req, sizeof(*req)); if (req == NULL) @@ -712,11 +766,10 @@ int lov_prep_brw_set(struct obd_export *exp, struct obd_info *oinfo, continue; loi = oinfo->oi_md->lsm_oinfo[i]; - if (!lov->lov_tgts[loi->loi_ost_idx] || - !lov->lov_tgts[loi->loi_ost_idx]->ltd_active) { - CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx); - GOTO(out, rc = -EIO); - } + if (!lov_check_and_wait_active(lov, loi->loi_ost_idx)) { + CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx); + GOTO(out, rc = -EIO); + } OBD_ALLOC(req, sizeof(*req)); if (req == NULL) @@ -829,18 +882,17 @@ int lov_prep_getattr_set(struct obd_export *exp, struct obd_info *oinfo, set->set_oi = oinfo; for (i = 0; i < oinfo->oi_md->lsm_stripe_count; i++) { - struct lov_oinfo *loi; - struct lov_request *req; - - loi = oinfo->oi_md->lsm_oinfo[i]; - if (!lov->lov_tgts[loi->loi_ost_idx] || - !lov->lov_tgts[loi->loi_ost_idx]->ltd_active) { - CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx); - if (oinfo->oi_oa->o_valid & OBD_MD_FLEPOCH) - /* SOM requires all the OSTs to be active. */ - GOTO(out_set, rc = -EIO); - continue; - } + struct lov_oinfo *loi; + struct lov_request *req; + + loi = oinfo->oi_md->lsm_oinfo[i]; + if (!lov_check_and_wait_active(lov, loi->loi_ost_idx)) { + CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx); + if (oinfo->oi_oa->o_valid & OBD_MD_FLEPOCH) + /* SOM requires all the OSTs to be active. */ + GOTO(out_set, rc = -EIO); + continue; + } OBD_ALLOC(req, sizeof(*req)); if (req == NULL) @@ -911,15 +963,14 @@ int lov_prep_destroy_set(struct obd_export *exp, struct obd_info *oinfo, set->set_cookies = oti->oti_logcookies; for (i = 0; i < lsm->lsm_stripe_count; i++) { - struct lov_oinfo *loi; - struct lov_request *req; + struct lov_oinfo *loi; + struct lov_request *req; - loi = lsm->lsm_oinfo[i]; - if (!lov->lov_tgts[loi->loi_ost_idx] || - !lov->lov_tgts[loi->loi_ost_idx]->ltd_active) { - CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx); - continue; - } + loi = lsm->lsm_oinfo[i]; + if (!lov_check_and_wait_active(lov, loi->loi_ost_idx)) { + CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx); + continue; + } OBD_ALLOC(req, sizeof(*req)); if (req == NULL) @@ -1023,14 +1074,13 @@ int lov_prep_setattr_set(struct obd_export *exp, struct obd_info *oinfo, set->set_cookies = oti->oti_logcookies; for (i = 0; i < oinfo->oi_md->lsm_stripe_count; i++) { - struct lov_oinfo *loi = oinfo->oi_md->lsm_oinfo[i]; - struct lov_request *req; + struct lov_oinfo *loi = oinfo->oi_md->lsm_oinfo[i]; + struct lov_request *req; - if (!lov->lov_tgts[loi->loi_ost_idx] || - !lov->lov_tgts[loi->loi_ost_idx]->ltd_active) { - CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx); - continue; - } + if (!lov_check_and_wait_active(lov, loi->loi_ost_idx)) { + CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx); + continue; + } OBD_ALLOC(req, sizeof(*req)); if (req == NULL) @@ -1157,11 +1207,10 @@ int lov_prep_punch_set(struct obd_export *exp, struct obd_info *oinfo, &rs, &re)) continue; - if (!lov->lov_tgts[loi->loi_ost_idx] || - !lov->lov_tgts[loi->loi_ost_idx]->ltd_active) { - CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx); - GOTO(out_set, rc = -EIO); - } + if (!lov_check_and_wait_active(lov, loi->loi_ost_idx)) { + CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx); + GOTO(out_set, rc = -EIO); + } OBD_ALLOC(req, sizeof(*req)); if (req == NULL) @@ -1247,15 +1296,14 @@ int lov_prep_sync_set(struct obd_export *exp, struct obd_info *oinfo, set->set_oi = oinfo; for (i = 0; i < oinfo->oi_md->lsm_stripe_count; i++) { - struct lov_oinfo *loi = oinfo->oi_md->lsm_oinfo[i]; - struct lov_request *req; - obd_off rs, re; + struct lov_oinfo *loi = oinfo->oi_md->lsm_oinfo[i]; + struct lov_request *req; + obd_off rs, re; - if (!lov->lov_tgts[loi->loi_ost_idx] || - !lov->lov_tgts[loi->loi_ost_idx]->ltd_active) { - CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx); - continue; - } + if (!lov_check_and_wait_active(lov, loi->loi_ost_idx)) { + CDEBUG(D_HA, "lov idx %d inactive\n", loi->loi_ost_idx); + continue; + } if (!lov_stripe_intersects(oinfo->oi_md, i, start, end, &rs, &re)) @@ -1478,13 +1526,14 @@ int lov_prep_statfs_set(struct obd_device *obd, struct obd_info *oinfo, /* We only get block data from the OBD */ for (i = 0; i < lov->desc.ld_tgt_count; i++) { - struct lov_request *req; - - if (!lov->lov_tgts[i] || (!lov->lov_tgts[i]->ltd_active - && (oinfo->oi_flags & OBD_STATFS_NODELAY))) { - CDEBUG(D_HA, "lov idx %d inactive\n", i); - continue; - } + struct lov_request *req; + + if (lov->lov_tgts[i] == NULL || + (!lov_check_and_wait_active(lov, i) && + (oinfo->oi_flags & OBD_STATFS_NODELAY))) { + CDEBUG(D_HA, "lov idx %d inactive\n", i); + continue; + } /* skip targets that have been explicitely disabled by the * administrator */ diff --git a/lustre/ptlrpc/import.c b/lustre/ptlrpc/import.c index 7eaee96..cdb91cf 100644 --- a/lustre/ptlrpc/import.c +++ b/lustre/ptlrpc/import.c @@ -771,6 +771,7 @@ static int ptlrpc_connect_interpret(const struct lu_env *env, spin_lock(&imp->imp_lock); if (imp->imp_state == LUSTRE_IMP_CLOSED) { + imp->imp_connect_tried = 1; spin_unlock(&imp->imp_lock); RETURN(0); } @@ -985,17 +986,18 @@ static int ptlrpc_connect_interpret(const struct lu_env *env, } finish: - rc = ptlrpc_import_recovery_state_machine(imp); - if (rc != 0) { - if (rc == -ENOTCONN) { - CDEBUG(D_HA, "evicted/aborted by %s@%s during recovery;" - "invalidating and reconnecting\n", - obd2cli_tgt(imp->imp_obd), - imp->imp_connection->c_remote_uuid.uuid); - ptlrpc_connect_import(imp); - RETURN(0); - } - } else { + rc = ptlrpc_import_recovery_state_machine(imp); + if (rc != 0) { + if (rc == -ENOTCONN) { + CDEBUG(D_HA, "evicted/aborted by %s@%s during recovery;" + "invalidating and reconnecting\n", + obd2cli_tgt(imp->imp_obd), + imp->imp_connection->c_remote_uuid.uuid); + ptlrpc_connect_import(imp); + imp->imp_connect_tried = 1; + RETURN(0); + } + } else { spin_lock(&imp->imp_lock); cfs_list_del(&imp->imp_conn_current->oic_item); @@ -1133,6 +1135,8 @@ finish: } out: + imp->imp_connect_tried = 1; + if (rc != 0) { IMPORT_SET_STATE(imp, LUSTRE_IMP_DISCON); if (rc == -EACCES) { -- 1.8.3.1