From: yangsheng Date: Tue, 9 Feb 2010 21:06:02 +0000 (+0800) Subject: b=21411 Improvement for AT to handle timeout more accurate. X-Git-Tag: 1.10.0.37~16 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=4b0e453a36f51004af2edea034fc5ae27afcd75b;p=fs%2Flustre-release.git b=21411 Improvement for AT to handle timeout more accurate. i=nathan i=tappro --- diff --git a/lustre/include/lustre_import.h b/lustre/include/lustre_import.h index 98124db..ebcc46f 100644 --- a/lustre/include/lustre_import.h +++ b/lustre/include/lustre_import.h @@ -238,7 +238,7 @@ static inline void at_init(struct adaptive_timeout *at, int val, int flags) { static inline int at_get(struct adaptive_timeout *at) { return at->at_current; } -int at_add(struct adaptive_timeout *at, unsigned int val); +int at_measured(struct adaptive_timeout *at, unsigned int val); int import_at_get_index(struct obd_import *imp, int portal); extern unsigned int at_max; #define AT_OFF (at_max == 0) diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index d683bae..b0ac944 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -1387,8 +1387,8 @@ target_start_and_reset_recovery_timer(struct obd_device *obd, if (!new_client && service_time) /* Teach server about old server's estimates, as first guess * at how long new requests will take. */ - at_add(&req->rq_rqbd->rqbd_service->srv_at_estimate, - service_time); + at_measured(&req->rq_rqbd->rqbd_service->srv_at_estimate, + service_time); check_and_start_recovery_timer(obd); diff --git a/lustre/ldlm/ldlm_lockd.c b/lustre/ldlm/ldlm_lockd.c index 78375dd..b89b699 100644 --- a/lustre/ldlm/ldlm_lockd.c +++ b/lustre/ldlm/ldlm_lockd.c @@ -855,8 +855,8 @@ int ldlm_server_completion_ast(struct ldlm_lock *lock, int flags, void *data) /* Server-side enqueue wait time estimate, used in __ldlm_add_waiting_lock to set future enqueue timers */ if (total_enqueue_wait < ldlm_get_enq_timeout(lock)) - at_add(&lock->l_resource->lr_namespace->ns_at_estimate, - total_enqueue_wait); + at_measured(&lock->l_resource->lr_namespace->ns_at_estimate, + total_enqueue_wait); else /* bz18618. Don't add lock enqueue time we spend waiting for a previous callback to fail. Locks waiting legitimately will diff --git a/lustre/ldlm/ldlm_request.c b/lustre/ldlm/ldlm_request.c index 50377c9..7b55b01 100644 --- a/lustre/ldlm/ldlm_request.c +++ b/lustre/ldlm/ldlm_request.c @@ -140,7 +140,8 @@ static int ldlm_completion_tail(struct ldlm_lock *lock) CFS_DURATION_T"s", delay); /* Update our time estimate */ - at_add(&lock->l_resource->lr_namespace->ns_at_estimate, delay); + at_measured(&lock->l_resource->lr_namespace->ns_at_estimate, + delay); result = 0; } return result; diff --git a/lustre/ptlrpc/client.c b/lustre/ptlrpc/client.c index 4d81fe6..e02de9c 100644 --- a/lustre/ptlrpc/client.c +++ b/lustre/ptlrpc/client.c @@ -228,7 +228,7 @@ static void ptlrpc_at_adj_service(struct ptlrpc_request *req, idx = import_at_get_index(req->rq_import, req->rq_request_portal); /* max service estimates are tracked on the server side, so just keep minimal history here */ - oldse = at_add(&at->iat_service_estimate[idx], serv_est); + oldse = at_measured(&at->iat_service_estimate[idx], serv_est); if (oldse != 0) CDEBUG(D_ADAPTTO, "The RPC service estimate for %s ptl %d " "has changed from %d to %d\n", @@ -260,7 +260,7 @@ static void ptlrpc_at_adj_net_latency(struct ptlrpc_request *req, CFS_DURATION_T"\n", service_time, cfs_time_sub(now, req->rq_sent)); - oldnl = at_add(&at->iat_net_latency, nl); + oldnl = at_measured(&at->iat_net_latency, nl); if (oldnl != 0) CDEBUG(D_ADAPTTO, "The network latency for %s (nid %s) " "has changed from %d to %d\n", diff --git a/lustre/ptlrpc/import.c b/lustre/ptlrpc/import.c index 12a19db..d5a1e74 100644 --- a/lustre/ptlrpc/import.c +++ b/lustre/ptlrpc/import.c @@ -518,9 +518,9 @@ static int import_select_connection(struct obd_import *imp) !imp->imp_recon_bk /* not retrying */) { if (at_get(&imp->imp_at.iat_net_latency) < CONNECTION_SWITCH_MAX) { - at_add(&imp->imp_at.iat_net_latency, - at_get(&imp->imp_at.iat_net_latency) + - CONNECTION_SWITCH_INC); + at_measured(&imp->imp_at.iat_net_latency, + at_get(&imp->imp_at.iat_net_latency) + + CONNECTION_SWITCH_INC); } LASSERT(imp_conn->oic_last_attempt); CWARN("%s: tried all connections, increasing latency to %ds\n", @@ -1487,7 +1487,7 @@ extern unsigned int at_min, at_max, at_history; This gives us a max of the last binlimit*AT_BINS secs without the storage, but still smoothing out a return to normalcy from a slow response. (E.g. remember the maximum latency in each minute of the last 4 minutes.) */ -int at_add(struct adaptive_timeout *at, unsigned int val) +int at_measured(struct adaptive_timeout *at, unsigned int val) { unsigned int old = at->at_current; time_t now = cfs_time_current_sec(); diff --git a/lustre/ptlrpc/niobuf.c b/lustre/ptlrpc/niobuf.c index 8be8f00..571cf17 100644 --- a/lustre/ptlrpc/niobuf.c +++ b/lustre/ptlrpc/niobuf.c @@ -350,7 +350,7 @@ static void ptlrpc_at_set_reply(struct ptlrpc_request *req, int flags) MSG_REQ_REPLAY_DONE | MSG_LOCK_REPLAY_DONE))) { /* early replies, errors and recovery requests don't count * toward our service time estimate */ - int oldse = at_add(&svc->srv_at_estimate, service_time); + int oldse = at_measured(&svc->srv_at_estimate, service_time); if (oldse != 0) DEBUG_REQ(D_ADAPTTO, req, "svc %s changed estimate from %d to %d", diff --git a/lustre/ptlrpc/service.c b/lustre/ptlrpc/service.c index b72848a..f6e6a65 100644 --- a/lustre/ptlrpc/service.c +++ b/lustre/ptlrpc/service.c @@ -974,29 +974,34 @@ static int ptlrpc_at_send_early_reply(struct ptlrpc_request *req) if (req->rq_export && lustre_msg_get_flags(req->rq_reqmsg) & (MSG_REPLAY | MSG_REQ_REPLAY_DONE | MSG_LOCK_REPLAY_DONE)) { - /** - * Use at_extra as early reply period for recovery requests but - * make sure it is not bigger than recovery time / 4 - */ - at_add(&svc->srv_at_estimate, - min(at_extra, - req->rq_export->exp_obd->obd_recovery_timeout / 4)); + /* During recovery, we don't want to send too many early + * replies, but on the other hand we want to make sure the + * client has enough time to resend if the rpc is lost. So + * during the recovery period send at least 4 early replies, + * spacing them every at_extra if we can. at_estimate should + * always equal this fixed value during recovery. */ + at_measured(&svc->srv_at_estimate, min(at_extra, + req->rq_export->exp_obd->obd_recovery_timeout / 4)); } else { /* Fake our processing time into the future to ask the clients * for some extra amount of time */ - at_add(&svc->srv_at_estimate, at_extra); + at_measured(&svc->srv_at_estimate, at_extra + + cfs_time_current_sec() - + req->rq_arrival_time.tv_sec); + + /* Check to see if we've actually increased the deadline - + * we may be past adaptive_max */ + if (req->rq_deadline >= req->rq_arrival_time.tv_sec + + at_get(&svc->srv_at_estimate)) { + DEBUG_REQ(D_WARNING, req, "Couldn't add any time " + "(%ld/%ld), not sending early reply\n", + olddl, req->rq_arrival_time.tv_sec + + at_get(&svc->srv_at_estimate) - + cfs_time_current_sec()); + RETURN(-ETIMEDOUT); + } } - newdl = cfs_time_current_sec() + at_get(&svc->srv_at_estimate); - if (req->rq_deadline >= newdl) { - /* We're not adding any time, no need to send an early reply - (e.g. maybe at adaptive_max) */ - DEBUG_REQ(D_WARNING, req, "Couldn't add any time (" - CFS_DURATION_T"/"CFS_DURATION_T"), " - "not sending early reply\n", olddl, - cfs_time_sub(newdl, cfs_time_current_sec())); - RETURN(-ETIMEDOUT); - } OBD_ALLOC(reqcopy, sizeof *reqcopy); if (reqcopy == NULL)