Whamcloud - gitweb
Branch HEAD
[fs/lustre-release.git] / lustre / ptlrpc / service.c
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  *
4  * GPL HEADER START
5  *
6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License version 2 only,
10  * as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope that it will be useful, but
13  * WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * General Public License version 2 for more details (a copy is included
16  * in the LICENSE file that accompanied this code).
17  *
18  * You should have received a copy of the GNU General Public License
19  * version 2 along with this program; If not, see
20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
21  *
22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
23  * CA 95054 USA or visit www.sun.com if you need additional information or
24  * have any questions.
25  *
26  * GPL HEADER END
27  */
28 /*
29  * Copyright  2008 Sun Microsystems, Inc. All rights reserved
30  * Use is subject to license terms.
31  */
32 /*
33  * This file is part of Lustre, http://www.lustre.org/
34  * Lustre is a trademark of Sun Microsystems, Inc.
35  */
36
37 #define DEBUG_SUBSYSTEM S_RPC
38 #ifndef __KERNEL__
39 #include <liblustre.h>
40 #endif
41 #include <obd_support.h>
42 #include <obd_class.h>
43 #include <lustre_net.h>
44 #include <lu_object.h>
45 #include <lnet/types.h>
46 #include "ptlrpc_internal.h"
47
48 /* The following are visible and mutable through /sys/module/ptlrpc */
49 int test_req_buffer_pressure = 0;
50 CFS_MODULE_PARM(test_req_buffer_pressure, "i", int, 0444,
51                 "set non-zero to put pressure on request buffer pools");
52 unsigned int at_min = 0;
53 CFS_MODULE_PARM(at_min, "i", int, 0644,
54                 "Adaptive timeout minimum (sec)");
55 unsigned int at_max = 600;
56 EXPORT_SYMBOL(at_max);
57 CFS_MODULE_PARM(at_max, "i", int, 0644,
58                 "Adaptive timeout maximum (sec)");
59 unsigned int at_history = 600;
60 CFS_MODULE_PARM(at_history, "i", int, 0644,
61                 "Adaptive timeouts remember the slowest event that took place "
62                 "within this period (sec)");
63 static int at_early_margin = 5;
64 CFS_MODULE_PARM(at_early_margin, "i", int, 0644,
65                 "How soon before an RPC deadline to send an early reply");
66 static int at_extra = 30;
67 CFS_MODULE_PARM(at_extra, "i", int, 0644,
68                 "How much extra time to give with each early reply");
69
70
71 /* forward ref */
72 static int ptlrpc_server_post_idle_rqbds (struct ptlrpc_service *svc);
73
74 static CFS_LIST_HEAD(ptlrpc_all_services);
75 spinlock_t ptlrpc_all_services_lock;
76
77 static char *
78 ptlrpc_alloc_request_buffer (int size)
79 {
80         char *ptr;
81
82         if (size > SVC_BUF_VMALLOC_THRESHOLD)
83                 OBD_VMALLOC(ptr, size);
84         else
85                 OBD_ALLOC(ptr, size);
86
87         return (ptr);
88 }
89
90 static void
91 ptlrpc_free_request_buffer (char *ptr, int size)
92 {
93         if (size > SVC_BUF_VMALLOC_THRESHOLD)
94                 OBD_VFREE(ptr, size);
95         else
96                 OBD_FREE(ptr, size);
97 }
98
99 struct ptlrpc_request_buffer_desc *
100 ptlrpc_alloc_rqbd (struct ptlrpc_service *svc)
101 {
102         struct ptlrpc_request_buffer_desc *rqbd;
103
104         OBD_ALLOC_PTR(rqbd);
105         if (rqbd == NULL)
106                 return (NULL);
107
108         rqbd->rqbd_service = svc;
109         rqbd->rqbd_refcount = 0;
110         rqbd->rqbd_cbid.cbid_fn = request_in_callback;
111         rqbd->rqbd_cbid.cbid_arg = rqbd;
112         CFS_INIT_LIST_HEAD(&rqbd->rqbd_reqs);
113         rqbd->rqbd_buffer = ptlrpc_alloc_request_buffer(svc->srv_buf_size);
114
115         if (rqbd->rqbd_buffer == NULL) {
116                 OBD_FREE_PTR(rqbd);
117                 return (NULL);
118         }
119
120         spin_lock(&svc->srv_lock);
121         list_add(&rqbd->rqbd_list, &svc->srv_idle_rqbds);
122         svc->srv_nbufs++;
123         spin_unlock(&svc->srv_lock);
124
125         return (rqbd);
126 }
127
128 void
129 ptlrpc_free_rqbd (struct ptlrpc_request_buffer_desc *rqbd)
130 {
131         struct ptlrpc_service *svc = rqbd->rqbd_service;
132
133         LASSERT (rqbd->rqbd_refcount == 0);
134         LASSERT (list_empty(&rqbd->rqbd_reqs));
135
136         spin_lock(&svc->srv_lock);
137         list_del(&rqbd->rqbd_list);
138         svc->srv_nbufs--;
139         spin_unlock(&svc->srv_lock);
140
141         ptlrpc_free_request_buffer (rqbd->rqbd_buffer, svc->srv_buf_size);
142         OBD_FREE_PTR(rqbd);
143 }
144
145 int
146 ptlrpc_grow_req_bufs(struct ptlrpc_service *svc)
147 {
148         struct ptlrpc_request_buffer_desc *rqbd;
149         int                                i;
150
151         CDEBUG(D_RPCTRACE, "%s: allocate %d new %d-byte reqbufs (%d/%d left)\n",
152                svc->srv_name, svc->srv_nbuf_per_group, svc->srv_buf_size,
153                svc->srv_nrqbd_receiving, svc->srv_nbufs);
154         for (i = 0; i < svc->srv_nbuf_per_group; i++) {
155                 rqbd = ptlrpc_alloc_rqbd(svc);
156
157                 if (rqbd == NULL) {
158                         CERROR ("%s: Can't allocate request buffer\n",
159                                 svc->srv_name);
160                         return (-ENOMEM);
161                 }
162
163                 if (ptlrpc_server_post_idle_rqbds(svc) < 0)
164                         return (-EAGAIN);
165         }
166
167         return (0);
168 }
169
170 void
171 ptlrpc_save_lock (struct ptlrpc_request *req,
172                   struct lustre_handle *lock, int mode)
173 {
174         struct ptlrpc_reply_state *rs = req->rq_reply_state;
175         int                        idx;
176
177         LASSERT(rs != NULL);
178         LASSERT(rs->rs_nlocks < RS_MAX_LOCKS);
179
180         idx = rs->rs_nlocks++;
181         rs->rs_locks[idx] = *lock;
182         rs->rs_modes[idx] = mode;
183         rs->rs_difficult = 1;
184 }
185
186 void
187 ptlrpc_schedule_difficult_reply (struct ptlrpc_reply_state *rs)
188 {
189         struct ptlrpc_service *svc = rs->rs_service;
190
191 #ifdef CONFIG_SMP
192         LASSERT (spin_is_locked (&svc->srv_lock));
193 #endif
194         LASSERT (rs->rs_difficult);
195         rs->rs_scheduled_ever = 1;              /* flag any notification attempt */
196
197         if (rs->rs_scheduled)                   /* being set up or already notified */
198                 return;
199
200         rs->rs_scheduled = 1;
201         list_del (&rs->rs_list);
202         list_add (&rs->rs_list, &svc->srv_reply_queue);
203         cfs_waitq_signal (&svc->srv_waitq);
204 }
205
206 void
207 ptlrpc_commit_replies (struct obd_device *obd)
208 {
209         struct list_head   *tmp;
210         struct list_head   *nxt;
211
212         /* Find any replies that have been committed and get their service
213          * to attend to complete them. */
214
215         /* CAVEAT EMPTOR: spinlock ordering!!! */
216         spin_lock(&obd->obd_uncommitted_replies_lock);
217
218         list_for_each_safe (tmp, nxt, &obd->obd_uncommitted_replies) {
219                 struct ptlrpc_reply_state *rs =
220                         list_entry(tmp, struct ptlrpc_reply_state, rs_obd_list);
221
222                 LASSERT (rs->rs_difficult);
223
224                 if (rs->rs_transno <= obd->obd_last_committed) {
225                         struct ptlrpc_service *svc = rs->rs_service;
226
227                         spin_lock (&svc->srv_lock);
228                         list_del_init (&rs->rs_obd_list);
229                         ptlrpc_schedule_difficult_reply (rs);
230                         spin_unlock (&svc->srv_lock);
231                 }
232         }
233
234         spin_unlock(&obd->obd_uncommitted_replies_lock);
235 }
236
237 static int
238 ptlrpc_server_post_idle_rqbds (struct ptlrpc_service *svc)
239 {
240         struct ptlrpc_request_buffer_desc *rqbd;
241         int                                rc;
242         int                                posted = 0;
243
244         for (;;) {
245                 spin_lock(&svc->srv_lock);
246
247                 if (list_empty (&svc->srv_idle_rqbds)) {
248                         spin_unlock(&svc->srv_lock);
249                         return (posted);
250                 }
251
252                 rqbd = list_entry(svc->srv_idle_rqbds.next,
253                                   struct ptlrpc_request_buffer_desc,
254                                   rqbd_list);
255                 list_del (&rqbd->rqbd_list);
256
257                 /* assume we will post successfully */
258                 svc->srv_nrqbd_receiving++;
259                 list_add (&rqbd->rqbd_list, &svc->srv_active_rqbds);
260
261                 spin_unlock(&svc->srv_lock);
262
263                 rc = ptlrpc_register_rqbd(rqbd);
264                 if (rc != 0)
265                         break;
266
267                 posted = 1;
268         }
269
270         spin_lock(&svc->srv_lock);
271
272         svc->srv_nrqbd_receiving--;
273         list_del(&rqbd->rqbd_list);
274         list_add_tail(&rqbd->rqbd_list, &svc->srv_idle_rqbds);
275
276         /* Don't complain if no request buffers are posted right now; LNET
277          * won't drop requests because we set the portal lazy! */
278
279         spin_unlock(&svc->srv_lock);
280
281         return (-1);
282 }
283
284 struct ptlrpc_service *ptlrpc_init_svc_conf(struct ptlrpc_service_conf *c,
285                                             svc_handler_t h, char *name,
286                                             struct proc_dir_entry *proc_entry,
287                                             svcreq_printfn_t prntfn,
288                                             char *threadname)
289 {
290         return ptlrpc_init_svc(c->psc_nbufs, c->psc_bufsize,
291                                c->psc_max_req_size, c->psc_max_reply_size,
292                                c->psc_req_portal, c->psc_rep_portal,
293                                c->psc_watchdog_factor,
294                                h, name, proc_entry,
295                                prntfn, c->psc_min_threads, c->psc_max_threads,
296                                threadname, c->psc_ctx_tags);
297 }
298 EXPORT_SYMBOL(ptlrpc_init_svc_conf);
299
300 static void ptlrpc_at_timer(unsigned long castmeharder)
301 {
302         struct ptlrpc_service *svc = (struct ptlrpc_service *)castmeharder;
303         svc->srv_at_check = 1;
304         svc->srv_at_checktime = cfs_time_current();
305         cfs_waitq_signal(&svc->srv_waitq);
306 }
307
308 /* @threadname should be 11 characters or less - 3 will be added on */
309 struct ptlrpc_service *
310 ptlrpc_init_svc(int nbufs, int bufsize, int max_req_size, int max_reply_size,
311                 int req_portal, int rep_portal, int watchdog_factor,
312                 svc_handler_t handler, char *name,
313                 cfs_proc_dir_entry_t *proc_entry,
314                 svcreq_printfn_t svcreq_printfn,
315                 int min_threads, int max_threads,
316                 char *threadname, __u32 ctx_tags)
317 {
318         int                    rc;
319         struct ptlrpc_service *service;
320         ENTRY;
321
322         LASSERT (nbufs > 0);
323         LASSERT (bufsize >= max_req_size + SPTLRPC_MAX_PAYLOAD);
324         LASSERT (ctx_tags != 0);
325
326         OBD_ALLOC_PTR(service);
327         if (service == NULL)
328                 RETURN(NULL);
329
330         /* First initialise enough for early teardown */
331
332         service->srv_name = name;
333         spin_lock_init(&service->srv_lock);
334         CFS_INIT_LIST_HEAD(&service->srv_threads);
335         cfs_waitq_init(&service->srv_waitq);
336
337         service->srv_nbuf_per_group = test_req_buffer_pressure ? 1 : nbufs;
338         service->srv_max_req_size = max_req_size + SPTLRPC_MAX_PAYLOAD;
339         service->srv_buf_size = bufsize;
340         service->srv_rep_portal = rep_portal;
341         service->srv_req_portal = req_portal;
342         service->srv_watchdog_factor = watchdog_factor;
343         service->srv_handler = handler;
344         service->srv_request_history_print_fn = svcreq_printfn;
345         service->srv_request_seq = 1;           /* valid seq #s start at 1 */
346         service->srv_request_max_cull_seq = 0;
347         service->srv_threads_min = min_threads;
348         service->srv_threads_max = max_threads;
349         service->srv_thread_name = threadname;
350         service->srv_ctx_tags = ctx_tags;
351
352         rc = LNetSetLazyPortal(service->srv_req_portal);
353         LASSERT (rc == 0);
354
355         CFS_INIT_LIST_HEAD(&service->srv_request_queue);
356         CFS_INIT_LIST_HEAD(&service->srv_idle_rqbds);
357         CFS_INIT_LIST_HEAD(&service->srv_active_rqbds);
358         CFS_INIT_LIST_HEAD(&service->srv_history_rqbds);
359         CFS_INIT_LIST_HEAD(&service->srv_request_history);
360         CFS_INIT_LIST_HEAD(&service->srv_active_replies);
361         CFS_INIT_LIST_HEAD(&service->srv_reply_queue);
362         CFS_INIT_LIST_HEAD(&service->srv_free_rs_list);
363         cfs_waitq_init(&service->srv_free_rs_waitq);
364
365         spin_lock_init(&service->srv_at_lock);
366         CFS_INIT_LIST_HEAD(&service->srv_req_in_queue);
367         CFS_INIT_LIST_HEAD(&service->srv_at_list);
368         cfs_timer_init(&service->srv_at_timer, ptlrpc_at_timer, service);
369         /* At SOW, service time should be quick; 10s seems generous. If client
370            timeout is less than this, we'll be sending an early reply. */
371         at_init(&service->srv_at_estimate, 10, 0);
372
373         spin_lock (&ptlrpc_all_services_lock);
374         list_add (&service->srv_list, &ptlrpc_all_services);
375         spin_unlock (&ptlrpc_all_services_lock);
376
377         /* Now allocate the request buffers */
378         rc = ptlrpc_grow_req_bufs(service);
379         /* We shouldn't be under memory pressure at startup, so
380          * fail if we can't post all our buffers at this time. */
381         if (rc != 0)
382                 GOTO(failed, NULL);
383
384         /* Now allocate pool of reply buffers */
385         /* Increase max reply size to next power of two */
386         service->srv_max_reply_size = 1;
387         while (service->srv_max_reply_size <
388                max_reply_size + SPTLRPC_MAX_PAYLOAD)
389                 service->srv_max_reply_size <<= 1;
390
391         if (proc_entry != NULL)
392                 ptlrpc_lprocfs_register_service(proc_entry, service);
393
394         CDEBUG(D_NET, "%s: Started, listening on portal %d\n",
395                service->srv_name, service->srv_req_portal);
396
397         RETURN(service);
398 failed:
399         ptlrpc_unregister_service(service);
400         return NULL;
401 }
402
403 /**
404  * to actually free the request, must be called without holding svc_lock.
405  * note it's caller's responsibility to unlink req->rq_list.
406  */
407 static void ptlrpc_server_free_request(struct ptlrpc_request *req)
408 {
409         LASSERT(atomic_read(&req->rq_refcount) == 0);
410         LASSERT(list_empty(&req->rq_timed_list));
411
412          /* DEBUG_REQ() assumes the reply state of a request with a valid
413           * ref will not be destroyed until that reference is dropped. */
414         ptlrpc_req_drop_rs(req);
415
416         sptlrpc_svc_ctx_decref(req);
417
418         if (req != &req->rq_rqbd->rqbd_req) {
419                 /* NB request buffers use an embedded
420                  * req if the incoming req unlinked the
421                  * MD; this isn't one of them! */
422                 OBD_FREE(req, sizeof(*req));
423         }
424 }
425
426 /**
427  * drop a reference count of the request. if it reaches 0, we either
428  * put it into history list, or free it immediately.
429  */
430 static void ptlrpc_server_drop_request(struct ptlrpc_request *req)
431 {
432         struct ptlrpc_request_buffer_desc *rqbd = req->rq_rqbd;
433         struct ptlrpc_service             *svc = rqbd->rqbd_service;
434         int                                refcount;
435         struct list_head                  *tmp;
436         struct list_head                  *nxt;
437
438         if (!atomic_dec_and_test(&req->rq_refcount))
439                 return;
440
441         spin_lock(&svc->srv_lock);
442
443         svc->srv_n_active_reqs--;
444         list_add(&req->rq_list, &rqbd->rqbd_reqs);
445
446         refcount = --(rqbd->rqbd_refcount);
447         if (refcount == 0) {
448                 /* request buffer is now idle: add to history */
449                 list_del(&rqbd->rqbd_list);
450                 list_add_tail(&rqbd->rqbd_list, &svc->srv_history_rqbds);
451                 svc->srv_n_history_rqbds++;
452
453                 /* cull some history?
454                  * I expect only about 1 or 2 rqbds need to be recycled here */
455                 while (svc->srv_n_history_rqbds > svc->srv_max_history_rqbds) {
456                         rqbd = list_entry(svc->srv_history_rqbds.next,
457                                           struct ptlrpc_request_buffer_desc,
458                                           rqbd_list);
459
460                         list_del(&rqbd->rqbd_list);
461                         svc->srv_n_history_rqbds--;
462
463                         /* remove rqbd's reqs from svc's req history while
464                          * I've got the service lock */
465                         list_for_each(tmp, &rqbd->rqbd_reqs) {
466                                 req = list_entry(tmp, struct ptlrpc_request,
467                                                  rq_list);
468                                 /* Track the highest culled req seq */
469                                 if (req->rq_history_seq >
470                                     svc->srv_request_max_cull_seq)
471                                         svc->srv_request_max_cull_seq =
472                                                 req->rq_history_seq;
473                                 list_del(&req->rq_history_list);
474                         }
475
476                         spin_unlock(&svc->srv_lock);
477
478                         list_for_each_safe(tmp, nxt, &rqbd->rqbd_reqs) {
479                                 req = list_entry(rqbd->rqbd_reqs.next,
480                                                  struct ptlrpc_request,
481                                                  rq_list);
482                                 list_del(&req->rq_list);
483                                 ptlrpc_server_free_request(req);
484                         }
485
486                         spin_lock(&svc->srv_lock);
487                         /*
488                          * now all reqs including the embedded req has been
489                          * disposed, schedule request buffer for re-use.
490                          */
491                         LASSERT(atomic_read(&rqbd->rqbd_req.rq_refcount) == 0);
492                         list_add_tail(&rqbd->rqbd_list, &svc->srv_idle_rqbds);
493                 }
494
495                 spin_unlock(&svc->srv_lock);
496         } else if (req->rq_reply_state && req->rq_reply_state->rs_prealloc) {
497                 /* If we are low on memory, we are not interested in history */
498                 list_del(&req->rq_list);
499                 list_del_init(&req->rq_history_list);
500                 spin_unlock(&svc->srv_lock);
501
502                 ptlrpc_server_free_request(req);
503         } else {
504                 spin_unlock(&svc->srv_lock);
505         }
506 }
507
508 /**
509  * to finish a request: stop sending more early replies, and release
510  * the request. should be called after we finished handling the request.
511  */
512 static void ptlrpc_server_finish_request(struct ptlrpc_request *req)
513 {
514         struct ptlrpc_service  *svc = req->rq_rqbd->rqbd_service;
515
516         if (req->rq_phase != RQ_PHASE_NEW) /* incorrect message magic */
517                 DEBUG_REQ(D_INFO, req, "free req");
518
519         spin_lock(&svc->srv_at_lock);
520         req->rq_sent_final = 1;
521         list_del_init(&req->rq_timed_list);
522         spin_unlock(&svc->srv_at_lock);
523
524         ptlrpc_server_drop_request(req);
525 }
526
527 /* This function makes sure dead exports are evicted in a timely manner.
528    This function is only called when some export receives a message (i.e.,
529    the network is up.) */
530 static void ptlrpc_update_export_timer(struct obd_export *exp, long extra_delay)
531 {
532         struct obd_export *oldest_exp;
533         time_t oldest_time;
534
535         ENTRY;
536
537         LASSERT(exp);
538
539         /* Compensate for slow machines, etc, by faking our request time
540            into the future.  Although this can break the strict time-ordering
541            of the list, we can be really lazy here - we don't have to evict
542            at the exact right moment.  Eventually, all silent exports
543            will make it to the top of the list. */
544         exp->exp_last_request_time = max(exp->exp_last_request_time,
545                                          cfs_time_current_sec() + extra_delay);
546
547         CDEBUG(D_HA, "updating export %s at "CFS_TIME_T" exp %p\n",
548                exp->exp_client_uuid.uuid,
549                exp->exp_last_request_time, exp);
550
551         /* exports may get disconnected from the chain even though the
552            export has references, so we must keep the spin lock while
553            manipulating the lists */
554         spin_lock(&exp->exp_obd->obd_dev_lock);
555
556         if (list_empty(&exp->exp_obd_chain_timed)) {
557                 /* this one is not timed */
558                 spin_unlock(&exp->exp_obd->obd_dev_lock);
559                 EXIT;
560                 return;
561         }
562
563         list_move_tail(&exp->exp_obd_chain_timed,
564                        &exp->exp_obd->obd_exports_timed);
565
566         oldest_exp = list_entry(exp->exp_obd->obd_exports_timed.next,
567                                 struct obd_export, exp_obd_chain_timed);
568         oldest_time = oldest_exp->exp_last_request_time;
569         spin_unlock(&exp->exp_obd->obd_dev_lock);
570
571         if (exp->exp_obd->obd_recovering) {
572                 /* be nice to everyone during recovery */
573                 EXIT;
574                 return;
575         }
576
577         /* Note - racing to start/reset the obd_eviction timer is safe */
578         if (exp->exp_obd->obd_eviction_timer == 0) {
579                 /* Check if the oldest entry is expired. */
580                 if (cfs_time_current_sec() > (oldest_time + PING_EVICT_TIMEOUT +
581                                               extra_delay)) {
582                         /* We need a second timer, in case the net was down and
583                          * it just came back. Since the pinger may skip every
584                          * other PING_INTERVAL (see note in ptlrpc_pinger_main),
585                          * we better wait for 3. */
586                         exp->exp_obd->obd_eviction_timer =
587                                 cfs_time_current_sec() + 3 * PING_INTERVAL;
588                         CDEBUG(D_HA, "%s: Think about evicting %s from "CFS_TIME_T"\n",
589                                exp->exp_obd->obd_name, obd_export_nid2str(exp),
590                                oldest_time);
591                 }
592         } else {
593                 if (cfs_time_current_sec() >
594                     (exp->exp_obd->obd_eviction_timer + extra_delay)) {
595                         /* The evictor won't evict anyone who we've heard from
596                          * recently, so we don't have to check before we start
597                          * it. */
598                         if (!ping_evictor_wake(exp))
599                                 exp->exp_obd->obd_eviction_timer = 0;
600                 }
601         }
602
603         EXIT;
604 }
605
606 static int ptlrpc_check_req(struct ptlrpc_request *req)
607 {
608         if (unlikely(lustre_msg_get_conn_cnt(req->rq_reqmsg) <
609                      req->rq_export->exp_conn_cnt)) {
610                 DEBUG_REQ(D_ERROR, req,
611                           "DROPPING req from old connection %d < %d",
612                           lustre_msg_get_conn_cnt(req->rq_reqmsg),
613                           req->rq_export->exp_conn_cnt);
614                 return -EEXIST;
615         }
616         if (unlikely(req->rq_export->exp_obd &&
617                      req->rq_export->exp_obd->obd_fail)) {
618              /* Failing over, don't handle any more reqs, send
619                 error response instead. */
620                 CDEBUG(D_RPCTRACE, "Dropping req %p for failed obd %s\n",
621                        req, req->rq_export->exp_obd->obd_name);
622                 req->rq_status = -ENODEV;
623                 ptlrpc_error(req);
624                 return -ENODEV;
625         }
626
627         return 0;
628 }
629
630 static void ptlrpc_at_set_timer(struct ptlrpc_service *svc)
631 {
632         struct ptlrpc_request *rq;
633         __s32 next;
634
635         spin_lock(&svc->srv_at_lock);
636         if (list_empty(&svc->srv_at_list)) {
637                 cfs_timer_disarm(&svc->srv_at_timer);
638                 spin_unlock(&svc->srv_at_lock);
639                 return;
640         }
641
642         /* Set timer for closest deadline */
643         rq = list_entry(svc->srv_at_list.next, struct ptlrpc_request,
644                         rq_timed_list);
645         next = (__s32)(rq->rq_deadline - cfs_time_current_sec() -
646                        at_early_margin);
647         if (next <= 0)
648                 ptlrpc_at_timer((unsigned long)svc);
649         else
650                 cfs_timer_arm(&svc->srv_at_timer, cfs_time_shift(next));
651         spin_unlock(&svc->srv_at_lock);
652         CDEBUG(D_INFO, "armed %s at %+ds\n", svc->srv_name, next);
653 }
654
655 /* Add rpc to early reply check list */
656 static int ptlrpc_at_add_timed(struct ptlrpc_request *req)
657 {
658         struct ptlrpc_service *svc = req->rq_rqbd->rqbd_service;
659         struct ptlrpc_request *rq;
660         int found = 0;
661
662         if (AT_OFF)
663                 return(0);
664
665         if (req->rq_no_reply)
666                 return 0;
667
668         if ((lustre_msghdr_get_flags(req->rq_reqmsg) & MSGHDR_AT_SUPPORT) == 0)
669                 return(-ENOSYS);
670
671         spin_lock(&svc->srv_at_lock);
672
673         if (unlikely(req->rq_sent_final)) {
674                 spin_unlock(&svc->srv_at_lock);
675                 return 0;
676         }
677
678         LASSERT(list_empty(&req->rq_timed_list));
679         /* Add to sorted list.  Presumably latest rpcs will have the latest
680            deadlines, so search backward. */
681         list_for_each_entry_reverse(rq, &svc->srv_at_list, rq_timed_list) {
682                 if (req->rq_deadline >= rq->rq_deadline) {
683                         list_add(&req->rq_timed_list, &rq->rq_timed_list);
684                         found++;
685                         break;
686                 }
687         }
688         if (!found)
689                 /* Add to front if shortest deadline or list empty */
690                 list_add(&req->rq_timed_list, &svc->srv_at_list);
691
692         /* Check if we're the head of the list */
693         found = (svc->srv_at_list.next == &req->rq_timed_list);
694
695         spin_unlock(&svc->srv_at_lock);
696
697         if (found)
698                 ptlrpc_at_set_timer(svc);
699
700         return 0;
701 }
702
703 static int ptlrpc_at_send_early_reply(struct ptlrpc_request *req,
704                                       int extra_time)
705 {
706         struct ptlrpc_service *svc = req->rq_rqbd->rqbd_service;
707         struct ptlrpc_request *reqcopy;
708         struct lustre_msg *reqmsg;
709         cfs_duration_t olddl = req->rq_deadline - cfs_time_current_sec();
710         time_t newdl;
711         int rc;
712         ENTRY;
713
714         /* deadline is when the client expects us to reply, margin is the
715            difference between clients' and servers' expectations */
716         DEBUG_REQ(D_ADAPTTO, req,
717                   "%ssending early reply (deadline %+lds, margin %+lds) for "
718                   "%d+%d", AT_OFF ? "AT off - not " : "",
719                   olddl, olddl - at_get(&svc->srv_at_estimate),
720                   at_get(&svc->srv_at_estimate), extra_time);
721
722         if (AT_OFF)
723                 RETURN(0);
724
725         if (olddl < 0) {
726                 DEBUG_REQ(D_WARNING, req, "Already past deadline (%+lds), "
727                           "not sending early reply. Consider increasing "
728                           "at_early_margin (%d)?", olddl, at_early_margin);
729
730                 /* Return an error so we're not re-added to the timed list. */
731                 RETURN(-ETIMEDOUT);
732         }
733
734         if ((lustre_msghdr_get_flags(req->rq_reqmsg) & MSGHDR_AT_SUPPORT) == 0){
735                 DEBUG_REQ(D_INFO, req, "Wanted to ask client for more time, "
736                           "but no AT support");
737                 RETURN(-ENOSYS);
738         }
739
740         if (req->rq_export && req->rq_export->exp_in_recovery) {
741                 /* don't increase server estimates during recovery, and give
742                    clients the full recovery time. */
743                 newdl = cfs_time_current_sec() +
744                         req->rq_export->exp_obd->obd_recovery_timeout;
745         } else {
746                 if (extra_time) {
747                         /* Fake our processing time into the future to ask the
748                            clients for some extra amount of time */
749                         extra_time += cfs_time_current_sec() -
750                                 req->rq_arrival_time.tv_sec;
751                         at_add(&svc->srv_at_estimate, extra_time);
752                 }
753                 newdl = req->rq_arrival_time.tv_sec +
754                         at_get(&svc->srv_at_estimate);
755         }
756         if (req->rq_deadline >= newdl) {
757                 /* We're not adding any time, no need to send an early reply
758                    (e.g. maybe at adaptive_max) */
759                 DEBUG_REQ(D_WARNING, req, "Couldn't add any time ("
760                           CFS_DURATION_T"/"CFS_DURATION_T"), "
761                           "not sending early reply\n", olddl,
762                           cfs_time_sub(newdl, cfs_time_current_sec()));
763                 RETURN(-ETIMEDOUT);
764         }
765
766         OBD_ALLOC(reqcopy, sizeof *reqcopy);
767         if (reqcopy == NULL)
768                 RETURN(-ENOMEM);
769         OBD_ALLOC(reqmsg, req->rq_reqlen);
770         if (!reqmsg) {
771                 OBD_FREE(reqcopy, sizeof *reqcopy);
772                 RETURN(-ENOMEM);
773         }
774
775         *reqcopy = *req;
776         reqcopy->rq_reply_state = NULL;
777         reqcopy->rq_rep_swab_mask = 0;
778         reqcopy->rq_pack_bulk = 0;
779         reqcopy->rq_pack_udesc = 0;
780         reqcopy->rq_packed_final = 0;
781         sptlrpc_svc_ctx_addref(reqcopy);
782         /* We only need the reqmsg for the magic */
783         reqcopy->rq_reqmsg = reqmsg;
784         memcpy(reqmsg, req->rq_reqmsg, req->rq_reqlen);
785
786         if (req->rq_sent_final) {
787                 DEBUG_REQ(D_ADAPTTO, reqcopy, "Normal reply already sent out, "
788                           "abort sending early reply\n");
789                 GOTO(out, rc = 0);
790         }
791
792         /* Connection ref */
793         reqcopy->rq_export = class_conn2export(
794                                      lustre_msg_get_handle(reqcopy->rq_reqmsg));
795         if (reqcopy->rq_export == NULL)
796                 GOTO(out, rc = -ENODEV);
797
798         /* RPC ref */
799         class_export_rpc_get(reqcopy->rq_export);
800         if (reqcopy->rq_export->exp_obd &&
801             reqcopy->rq_export->exp_obd->obd_fail)
802                 GOTO(out_put, rc = -ENODEV);
803
804         rc = lustre_pack_reply_flags(reqcopy, 1, NULL, NULL, LPRFL_EARLY_REPLY);
805         if (rc)
806                 GOTO(out_put, rc);
807
808         rc = ptlrpc_send_reply(reqcopy, PTLRPC_REPLY_EARLY);
809
810         if (!rc) {
811                 /* Adjust our own deadline to what we told the client */
812                 req->rq_deadline = newdl;
813                 req->rq_early_count++; /* number sent, server side */
814         } else {
815                 DEBUG_REQ(D_ERROR, req, "Early reply send failed %d", rc);
816         }
817
818         /* Free the (early) reply state from lustre_pack_reply.
819            (ptlrpc_send_reply takes it's own rs ref, so this is safe here) */
820         ptlrpc_req_drop_rs(reqcopy);
821
822 out_put:
823         class_export_rpc_put(reqcopy->rq_export);
824         class_export_put(reqcopy->rq_export);
825 out:
826         sptlrpc_svc_ctx_decref(reqcopy);
827         OBD_FREE(reqmsg, req->rq_reqlen);
828         OBD_FREE(reqcopy, sizeof *reqcopy);
829         RETURN(rc);
830 }
831
832 /* Send early replies to everybody expiring within at_early_margin
833    asking for at_extra time */
834 static int ptlrpc_at_check_timed(struct ptlrpc_service *svc)
835 {
836         struct ptlrpc_request *rq, *n;
837         struct list_head work_list;
838         time_t now = cfs_time_current_sec();
839         cfs_duration_t delay;
840         int first, counter = 0;
841         ENTRY;
842
843         spin_lock(&svc->srv_at_lock);
844         if (svc->srv_at_check == 0) {
845                 spin_unlock(&svc->srv_at_lock);
846                 RETURN(0);
847         }
848         delay = cfs_time_sub(cfs_time_current(), svc->srv_at_checktime);
849         svc->srv_at_check = 0;
850
851         if (list_empty(&svc->srv_at_list)) {
852                 spin_unlock(&svc->srv_at_lock);
853                 RETURN(0);
854         }
855
856         /* The timer went off, but maybe the nearest rpc already completed. */
857         rq = list_entry(svc->srv_at_list.next, struct ptlrpc_request,
858                         rq_timed_list);
859         first = (int)(rq->rq_deadline - now);
860         if (first > at_early_margin) {
861                 /* We've still got plenty of time.  Reset the timer. */
862                 spin_unlock(&svc->srv_at_lock);
863                 ptlrpc_at_set_timer(svc);
864                 RETURN(0);
865         }
866
867         /* We're close to a timeout, and we don't know how much longer the
868            server will take. Send early replies to everyone expiring soon. */
869         CFS_INIT_LIST_HEAD(&work_list);
870         list_for_each_entry_safe(rq, n, &svc->srv_at_list, rq_timed_list) {
871                 if (rq->rq_deadline <= now + at_early_margin) {
872                         list_move_tail(&rq->rq_timed_list, &work_list);
873                         counter++;
874                 } else {
875                         break;
876                 }
877         }
878
879         spin_unlock(&svc->srv_at_lock);
880
881         /* we have a new earliest deadline, restart the timer */
882         ptlrpc_at_set_timer(svc);
883
884         CDEBUG(D_ADAPTTO, "timeout in %+ds, asking for %d secs on %d early "
885                "replies\n", first, at_extra, counter);
886         if (first < 0) {
887                 /* We're already past request deadlines before we even get a
888                    chance to send early replies */
889                 LCONSOLE_WARN("%s: This server is not able to keep up with "
890                               "request traffic (cpu-bound).\n", svc->srv_name);
891                 CWARN("earlyQ=%d reqQ=%d recA=%d, svcEst=%d, "
892                       "delay="CFS_DURATION_T"(jiff)\n",
893                       counter, svc->srv_n_queued_reqs, svc->srv_n_active_reqs,
894                       at_get(&svc->srv_at_estimate), delay);
895         }
896
897         /* ptlrpc_server_finish_request may delete an entry out of
898          * the work list */
899         spin_lock(&svc->srv_at_lock);
900         while (!list_empty(&work_list)) {
901                 rq = list_entry(work_list.next, struct ptlrpc_request,
902                                 rq_timed_list);
903                 list_del_init(&rq->rq_timed_list);
904                 /* if the entry is still in the worklist, it hasn't been
905                    deleted, and is safe to take a ref to keep the req around */
906                 atomic_inc(&rq->rq_refcount);
907                 spin_unlock(&svc->srv_at_lock);
908
909                 if (ptlrpc_at_send_early_reply(rq, at_extra) == 0)
910                         ptlrpc_at_add_timed(rq);
911
912                 ptlrpc_server_drop_request(rq);
913                 spin_lock(&svc->srv_at_lock);
914         }
915         spin_unlock(&svc->srv_at_lock);
916
917         RETURN(0);
918 }
919
920 /* Handle freshly incoming reqs, add to timed early reply list,
921    pass on to regular request queue */
922 static int
923 ptlrpc_server_handle_req_in(struct ptlrpc_service *svc)
924 {
925         struct ptlrpc_request *req;
926         __u32                  deadline;
927         int                    rc;
928         ENTRY;
929
930         LASSERT(svc);
931
932         spin_lock(&svc->srv_lock);
933         if (list_empty(&svc->srv_req_in_queue)) {
934                 spin_unlock(&svc->srv_lock);
935                 RETURN(0);
936         }
937
938         req = list_entry(svc->srv_req_in_queue.next,
939                          struct ptlrpc_request, rq_list);
940         list_del_init (&req->rq_list);
941         /* Consider this still a "queued" request as far as stats are
942            concerned */
943         spin_unlock(&svc->srv_lock);
944
945         /* go through security check/transform */
946         rc = sptlrpc_svc_unwrap_request(req);
947         switch (rc) {
948         case SECSVC_OK:
949                 break;
950         case SECSVC_COMPLETE:
951                 target_send_reply(req, 0, OBD_FAIL_MDS_ALL_REPLY_NET);
952                 goto err_req;
953         case SECSVC_DROP:
954                 goto err_req;
955         default:
956                 LBUG();
957         }
958
959         /* Clear request swab mask; this is a new request */
960         req->rq_req_swab_mask = 0;
961
962         rc = lustre_unpack_msg(req->rq_reqmsg, req->rq_reqlen);
963         if (rc < 0) {
964                 CERROR("error unpacking request: ptl %d from %s x"LPU64"\n",
965                        svc->srv_req_portal, libcfs_id2str(req->rq_peer),
966                        req->rq_xid);
967                 goto err_req;
968         }
969
970         if (rc > 0)
971                 lustre_set_req_swabbed(req, MSG_PTLRPC_HEADER_OFF);
972
973         rc = lustre_unpack_req_ptlrpc_body(req, MSG_PTLRPC_BODY_OFF);
974         if (rc) {
975                 CERROR ("error unpacking ptlrpc body: ptl %d from %s x"
976                         LPU64"\n", svc->srv_req_portal,
977                         libcfs_id2str(req->rq_peer), req->rq_xid);
978                 goto err_req;
979         }
980
981         rc = -EINVAL;
982         if (lustre_msg_get_type(req->rq_reqmsg) != PTL_RPC_MSG_REQUEST) {
983                 CERROR("wrong packet type received (type=%u) from %s\n",
984                        lustre_msg_get_type(req->rq_reqmsg),
985                        libcfs_id2str(req->rq_peer));
986                 goto err_req;
987         }
988
989         CDEBUG(D_NET, "got req "LPD64"\n", req->rq_xid);
990
991         req->rq_export = class_conn2export(
992                 lustre_msg_get_handle(req->rq_reqmsg));
993         if (req->rq_export) {
994                 rc = ptlrpc_check_req(req);
995                 if (rc == 0) {
996                         rc = sptlrpc_target_export_check(req->rq_export, req);
997                         if (rc)
998                                 DEBUG_REQ(D_ERROR, req, "DROPPING req with "
999                                           "illegal security flavor,");
1000                 }
1001
1002                 class_export_put(req->rq_export);
1003                 req->rq_export = NULL;
1004                 if (rc)
1005                         goto err_req;
1006         }
1007
1008         /* req_in handling should/must be fast */
1009         if (cfs_time_current_sec() - req->rq_arrival_time.tv_sec > 5)
1010                 DEBUG_REQ(D_WARNING, req, "Slow req_in handling "CFS_DURATION_T"s",
1011                           cfs_time_sub(cfs_time_current_sec(),
1012                                        req->rq_arrival_time.tv_sec));
1013
1014         /* Set rpc server deadline and add it to the timed list */
1015         deadline = (lustre_msghdr_get_flags(req->rq_reqmsg) &
1016                     MSGHDR_AT_SUPPORT) ?
1017                    /* The max time the client expects us to take */
1018                    lustre_msg_get_timeout(req->rq_reqmsg) : obd_timeout;
1019         req->rq_deadline = req->rq_arrival_time.tv_sec + deadline;
1020         if (unlikely(deadline == 0)) {
1021                 DEBUG_REQ(D_ERROR, req, "Dropping request with 0 timeout");
1022                 goto err_req;
1023         }
1024
1025         ptlrpc_at_add_timed(req);
1026
1027         /* Move it over to the request processing queue */
1028         spin_lock(&svc->srv_lock);
1029         list_add_tail(&req->rq_list, &svc->srv_request_queue);
1030         cfs_waitq_signal(&svc->srv_waitq);
1031         spin_unlock(&svc->srv_lock);
1032         RETURN(1);
1033
1034 err_req:
1035         spin_lock(&svc->srv_lock);
1036         svc->srv_n_queued_reqs--;
1037         svc->srv_n_active_reqs++;
1038         spin_unlock(&svc->srv_lock);
1039         ptlrpc_server_finish_request(req);
1040
1041         RETURN(1);
1042 }
1043
1044 static int
1045 ptlrpc_server_handle_request(struct ptlrpc_service *svc,
1046                              struct ptlrpc_thread *thread)
1047 {
1048         struct obd_export     *export = NULL;
1049         struct ptlrpc_request *request;
1050         struct timeval         work_start;
1051         struct timeval         work_end;
1052         long                   timediff;
1053         int                    rc;
1054         ENTRY;
1055
1056         LASSERT(svc);
1057
1058         spin_lock(&svc->srv_lock);
1059         if (unlikely(list_empty (&svc->srv_request_queue) ||
1060             (
1061 #ifndef __KERNEL__
1062              /* !@%$# liblustre only has 1 thread */
1063              svc->srv_n_difficult_replies != 0 &&
1064 #endif
1065              svc->srv_n_active_reqs >= (svc->srv_threads_running - 1)))) {
1066                  /* Don't handle regular requests in the last thread, in order               * re
1067                   * to handle difficult replies (which might block other threads)
1068                   * as well as handle any incoming reqs, early replies, etc.
1069                   * That means we always need at least 2 service threads. */
1070                 spin_unlock(&svc->srv_lock);
1071                 RETURN(0);
1072         }
1073
1074         request = list_entry (svc->srv_request_queue.next,
1075                               struct ptlrpc_request, rq_list);
1076         list_del_init (&request->rq_list);
1077         svc->srv_n_queued_reqs--;
1078         svc->srv_n_active_reqs++;
1079
1080         spin_unlock(&svc->srv_lock);
1081
1082         if(OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_DUMP_LOG))
1083                 libcfs_debug_dumplog();
1084
1085         do_gettimeofday(&work_start);
1086         timediff = cfs_timeval_sub(&work_start, &request->rq_arrival_time,NULL);
1087         if (likely(svc->srv_stats != NULL)) {
1088                 lprocfs_counter_add(svc->srv_stats, PTLRPC_REQWAIT_CNTR,
1089                                     timediff);
1090                 lprocfs_counter_add(svc->srv_stats, PTLRPC_REQQDEPTH_CNTR,
1091                                     svc->srv_n_queued_reqs);
1092                 lprocfs_counter_add(svc->srv_stats, PTLRPC_REQACTIVE_CNTR,
1093                                     svc->srv_n_active_reqs);
1094                 lprocfs_counter_add(svc->srv_stats, PTLRPC_TIMEOUT,
1095                                     at_get(&svc->srv_at_estimate));
1096         }
1097
1098         rc = lu_context_init(&request->rq_session, LCT_SESSION);
1099         if (rc) {
1100                 CERROR("Failure to initialize session: %d\n", rc);
1101                 goto out_req;
1102         }
1103         request->rq_session.lc_thread = thread;
1104         lu_context_enter(&request->rq_session);
1105
1106         CDEBUG(D_NET, "got req "LPU64"\n", request->rq_xid);
1107
1108         request->rq_svc_thread = thread;
1109         if (thread)
1110                 request->rq_svc_thread->t_env->le_ses = &request->rq_session;
1111
1112         request->rq_export = class_conn2export(
1113                                      lustre_msg_get_handle(request->rq_reqmsg));
1114
1115         if (likely(request->rq_export)) {
1116                 if (unlikely(ptlrpc_check_req(request)))
1117                         goto put_conn;
1118                 ptlrpc_update_export_timer(request->rq_export, timediff >> 19);
1119                 export = class_export_rpc_get(request->rq_export);
1120         }
1121
1122         /* Discard requests queued for longer than the deadline.
1123            The deadline is increased if we send an early reply. */
1124         if (cfs_time_current_sec() > request->rq_deadline) {
1125                 DEBUG_REQ(D_ERROR, request, "Dropping timed-out request from %s"
1126                           ": deadline "CFS_DURATION_T":"CFS_DURATION_T"s ago\n",
1127                           libcfs_id2str(request->rq_peer),
1128                           cfs_time_sub(request->rq_deadline,
1129                           request->rq_arrival_time.tv_sec),
1130                           cfs_time_sub(cfs_time_current_sec(),
1131                           request->rq_deadline));
1132                 goto put_rpc_export;
1133         }
1134
1135         request->rq_phase = RQ_PHASE_INTERPRET;
1136
1137         CDEBUG(D_RPCTRACE, "Handling RPC pname:cluuid+ref:pid:xid:nid:opc "
1138                "%s:%s+%d:%d:x"LPU64":%s:%d\n", cfs_curproc_comm(),
1139                (request->rq_export ?
1140                 (char *)request->rq_export->exp_client_uuid.uuid : "0"),
1141                (request->rq_export ?
1142                 atomic_read(&request->rq_export->exp_refcount) : -99),
1143                lustre_msg_get_status(request->rq_reqmsg), request->rq_xid,
1144                libcfs_id2str(request->rq_peer),
1145                lustre_msg_get_opc(request->rq_reqmsg));
1146
1147         OBD_FAIL_TIMEOUT_MS(OBD_FAIL_PTLRPC_PAUSE_REQ, obd_fail_val);
1148
1149         rc = svc->srv_handler(request);
1150
1151         request->rq_phase = RQ_PHASE_COMPLETE;
1152
1153         CDEBUG(D_RPCTRACE, "Handled RPC pname:cluuid+ref:pid:xid:nid:opc "
1154                "%s:%s+%d:%d:x"LPU64":%s:%d\n", cfs_curproc_comm(),
1155                (request->rq_export ?
1156                 (char *)request->rq_export->exp_client_uuid.uuid : "0"),
1157                (request->rq_export ?
1158                 atomic_read(&request->rq_export->exp_refcount) : -99),
1159                lustre_msg_get_status(request->rq_reqmsg), request->rq_xid,
1160                libcfs_id2str(request->rq_peer),
1161                lustre_msg_get_opc(request->rq_reqmsg));
1162
1163 put_rpc_export:
1164         if (export != NULL)
1165                 class_export_rpc_put(export);
1166 put_conn:
1167         if (likely(request->rq_export != NULL))
1168                 class_export_put(request->rq_export);
1169
1170         lu_context_exit(&request->rq_session);
1171         lu_context_fini(&request->rq_session);
1172
1173         if (unlikely(cfs_time_current_sec() > request->rq_deadline)) {
1174                 DEBUG_REQ(D_WARNING, request, "Request x"LPU64" took longer "
1175                           "than estimated ("CFS_DURATION_T":"CFS_DURATION_T"s);"
1176                           " client may timeout.",
1177                           request->rq_xid, cfs_time_sub(request->rq_deadline,
1178                           request->rq_arrival_time.tv_sec),
1179                           cfs_time_sub(cfs_time_current_sec(),
1180                           request->rq_deadline));
1181         }
1182
1183         do_gettimeofday(&work_end);
1184         timediff = cfs_timeval_sub(&work_end, &work_start, NULL);
1185         CDEBUG(D_RPCTRACE, "request x"LPU64" opc %u from %s processed in "
1186                "%ldus (%ldus total) trans "LPU64" rc %d/%d\n",
1187                request->rq_xid, lustre_msg_get_opc(request->rq_reqmsg),
1188                libcfs_id2str(request->rq_peer), timediff,
1189                cfs_timeval_sub(&work_end, &request->rq_arrival_time, NULL),
1190                request->rq_repmsg ? lustre_msg_get_transno(request->rq_repmsg) :
1191                request->rq_transno, request->rq_status,
1192                request->rq_repmsg ? lustre_msg_get_status(request->rq_repmsg):
1193                -999);
1194         if (likely(svc->srv_stats != NULL && request->rq_reqmsg != NULL)) {
1195                 __u32 op = lustre_msg_get_opc(request->rq_reqmsg);
1196                 int opc = opcode_offset(op);
1197                 if (opc > 0 && !(op == LDLM_ENQUEUE || op == MDS_REINT)) {
1198                         LASSERT(opc < LUSTRE_MAX_OPCODES);
1199                         lprocfs_counter_add(svc->srv_stats,
1200                                             opc + EXTRA_MAX_OPCODES,
1201                                             timediff);
1202                 }
1203         }
1204         if (unlikely(request->rq_early_count)) {
1205                 DEBUG_REQ(D_ADAPTTO, request,
1206                           "sent %d early replies before finishing in "
1207                           CFS_DURATION_T"s",
1208                           request->rq_early_count,
1209                           cfs_time_sub(work_end.tv_sec,
1210                           request->rq_arrival_time.tv_sec));
1211         }
1212
1213 out_req:
1214         ptlrpc_server_finish_request(request);
1215
1216         RETURN(1);
1217 }
1218
1219 static int
1220 ptlrpc_server_handle_reply (struct ptlrpc_service *svc)
1221 {
1222         struct ptlrpc_reply_state *rs;
1223         struct obd_export         *exp;
1224         struct obd_device         *obd;
1225         int                        nlocks;
1226         int                        been_handled;
1227         ENTRY;
1228
1229         spin_lock(&svc->srv_lock);
1230         if (list_empty (&svc->srv_reply_queue)) {
1231                 spin_unlock(&svc->srv_lock);
1232                 RETURN(0);
1233         }
1234
1235         rs = list_entry (svc->srv_reply_queue.next,
1236                          struct ptlrpc_reply_state, rs_list);
1237
1238         exp = rs->rs_export;
1239         obd = exp->exp_obd;
1240
1241         LASSERT (rs->rs_difficult);
1242         LASSERT (rs->rs_scheduled);
1243
1244         list_del_init (&rs->rs_list);
1245
1246         /* Disengage from notifiers carefully (lock order - irqrestore below!)*/
1247         spin_unlock(&svc->srv_lock);
1248
1249         spin_lock (&obd->obd_uncommitted_replies_lock);
1250         /* Noop if removed already */
1251         list_del_init (&rs->rs_obd_list);
1252         spin_unlock (&obd->obd_uncommitted_replies_lock);
1253
1254         spin_lock (&exp->exp_lock);
1255         /* Noop if removed already */
1256         list_del_init (&rs->rs_exp_list);
1257         spin_unlock (&exp->exp_lock);
1258
1259         spin_lock(&svc->srv_lock);
1260
1261         been_handled = rs->rs_handled;
1262         rs->rs_handled = 1;
1263
1264         nlocks = rs->rs_nlocks;                 /* atomic "steal", but */
1265         rs->rs_nlocks = 0;                      /* locks still on rs_locks! */
1266
1267         if (nlocks == 0 && !been_handled) {
1268                 /* If we see this, we should already have seen the warning
1269                  * in mds_steal_ack_locks()  */
1270                 CWARN("All locks stolen from rs %p x"LPD64".t"LPD64
1271                       " o%d NID %s\n",
1272                       rs,
1273                       rs->rs_xid, rs->rs_transno,
1274                       lustre_msg_get_opc(rs->rs_msg),
1275                       libcfs_nid2str(exp->exp_connection->c_peer.nid));
1276         }
1277
1278         if ((!been_handled && rs->rs_on_net) || nlocks > 0) {
1279                 spin_unlock(&svc->srv_lock);
1280
1281                 if (!been_handled && rs->rs_on_net) {
1282                         LNetMDUnlink(rs->rs_md_h);
1283                         /* Ignore return code; we're racing with
1284                          * completion... */
1285                 }
1286
1287                 while (nlocks-- > 0)
1288                         ldlm_lock_decref(&rs->rs_locks[nlocks],
1289                                          rs->rs_modes[nlocks]);
1290
1291                 spin_lock(&svc->srv_lock);
1292         }
1293
1294         rs->rs_scheduled = 0;
1295
1296         if (!rs->rs_on_net) {
1297                 /* Off the net */
1298                 svc->srv_n_difficult_replies--;
1299                 spin_unlock(&svc->srv_lock);
1300
1301                 class_export_put (exp);
1302                 rs->rs_export = NULL;
1303                 ptlrpc_rs_decref (rs);
1304                 atomic_dec (&svc->srv_outstanding_replies);
1305                 RETURN(1);
1306         }
1307
1308         /* still on the net; callback will schedule */
1309         spin_unlock(&svc->srv_lock);
1310         RETURN(1);
1311 }
1312
1313 #ifndef __KERNEL__
1314 /* FIXME make use of timeout later */
1315 int
1316 liblustre_check_services (void *arg)
1317 {
1318         int  did_something = 0;
1319         int  rc;
1320         struct list_head *tmp, *nxt;
1321         ENTRY;
1322
1323         /* I'm relying on being single threaded, not to have to lock
1324          * ptlrpc_all_services etc */
1325         list_for_each_safe (tmp, nxt, &ptlrpc_all_services) {
1326                 struct ptlrpc_service *svc =
1327                         list_entry (tmp, struct ptlrpc_service, srv_list);
1328
1329                 if (svc->srv_threads_running != 0)     /* I've recursed */
1330                         continue;
1331
1332                 /* service threads can block for bulk, so this limits us
1333                  * (arbitrarily) to recursing 1 stack frame per service.
1334                  * Note that the problem with recursion is that we have to
1335                  * unwind completely before our caller can resume. */
1336
1337                 svc->srv_threads_running++;
1338
1339                 do {
1340                         rc = ptlrpc_server_handle_req_in(svc);
1341                         rc |= ptlrpc_server_handle_reply(svc);
1342                         rc |= ptlrpc_at_check_timed(svc);
1343                         rc |= ptlrpc_server_handle_request(svc, NULL);
1344                         rc |= (ptlrpc_server_post_idle_rqbds(svc) > 0);
1345                         did_something |= rc;
1346                 } while (rc);
1347
1348                 svc->srv_threads_running--;
1349         }
1350
1351         RETURN(did_something);
1352 }
1353 #define ptlrpc_stop_all_threads(s) do {} while (0)
1354
1355 #else /* __KERNEL__ */
1356
1357 /* Don't use daemonize, it removes fs struct from new thread (bug 418) */
1358 void ptlrpc_daemonize(char *name)
1359 {
1360         struct fs_struct *fs = current->fs;
1361
1362         atomic_inc(&fs->count);
1363         cfs_daemonize(name);
1364         exit_fs(cfs_current());
1365         current->fs = fs;
1366         ll_set_fs_pwd(current->fs, init_task.fs->pwdmnt, init_task.fs->pwd);
1367 }
1368
1369 static void
1370 ptlrpc_check_rqbd_pool(struct ptlrpc_service *svc)
1371 {
1372         int avail = svc->srv_nrqbd_receiving;
1373         int low_water = test_req_buffer_pressure ? 0 :
1374                         svc->srv_nbuf_per_group/2;
1375
1376         /* NB I'm not locking; just looking. */
1377
1378         /* CAVEAT EMPTOR: We might be allocating buffers here because we've
1379          * allowed the request history to grow out of control.  We could put a
1380          * sanity check on that here and cull some history if we need the
1381          * space. */
1382
1383         if (avail <= low_water)
1384                 ptlrpc_grow_req_bufs(svc);
1385
1386         if (svc->srv_stats)
1387                 lprocfs_counter_add(svc->srv_stats, PTLRPC_REQBUF_AVAIL_CNTR,
1388                                     avail);
1389 }
1390
1391 static int
1392 ptlrpc_retry_rqbds(void *arg)
1393 {
1394         struct ptlrpc_service *svc = (struct ptlrpc_service *)arg;
1395
1396         svc->srv_rqbd_timeout = 0;
1397         return (-ETIMEDOUT);
1398 }
1399
1400 static int ptlrpc_main(void *arg)
1401 {
1402         struct ptlrpc_svc_data *data = (struct ptlrpc_svc_data *)arg;
1403         struct ptlrpc_service  *svc = data->svc;
1404         struct ptlrpc_thread   *thread = data->thread;
1405         struct obd_device      *dev = data->dev;
1406         struct ptlrpc_reply_state *rs;
1407         struct lc_watchdog     *watchdog;
1408 #ifdef WITH_GROUP_INFO
1409         struct group_info *ginfo = NULL;
1410 #endif
1411         struct lu_env env;
1412         int counter = 0, rc = 0;
1413         ENTRY;
1414
1415         ptlrpc_daemonize(data->name);
1416
1417 #if defined(HAVE_NODE_TO_CPUMASK) && defined(CONFIG_NUMA)
1418         /* we need to do this before any per-thread allocation is done so that
1419          * we get the per-thread allocations on local node.  bug 7342 */
1420         if (svc->srv_cpu_affinity) {
1421                 int cpu, num_cpu;
1422
1423                 for (cpu = 0, num_cpu = 0; cpu < num_possible_cpus(); cpu++) {
1424                         if (!cpu_online(cpu))
1425                                 continue;
1426                         if (num_cpu == thread->t_id % num_online_cpus())
1427                                 break;
1428                         num_cpu++;
1429                 }
1430                 set_cpus_allowed(cfs_current(), node_to_cpumask(cpu_to_node(cpu)));
1431         }
1432 #endif
1433
1434 #ifdef WITH_GROUP_INFO
1435         ginfo = groups_alloc(0);
1436         if (!ginfo) {
1437                 rc = -ENOMEM;
1438                 goto out;
1439         }
1440
1441         set_current_groups(ginfo);
1442         put_group_info(ginfo);
1443 #endif
1444
1445         if (svc->srv_init != NULL) {
1446                 rc = svc->srv_init(thread);
1447                 if (rc)
1448                         goto out;
1449         }
1450
1451         rc = lu_context_init(&env.le_ctx, svc->srv_ctx_tags);
1452         if (rc)
1453                 goto out_srv_fini;
1454
1455         thread->t_env = &env;
1456         env.le_ctx.lc_thread = thread;
1457
1458         /* Alloc reply state structure for this one */
1459         OBD_ALLOC_GFP(rs, svc->srv_max_reply_size, CFS_ALLOC_STD);
1460         if (!rs) {
1461                 rc = -ENOMEM;
1462                 goto out_srv_fini;
1463         }
1464
1465         /* Record that the thread is running */
1466         thread->t_flags = SVC_RUNNING;
1467         /*
1468          * wake up our creator. Note: @data is invalid after this point,
1469          * because it's allocated on ptlrpc_start_thread() stack.
1470          */
1471         cfs_waitq_signal(&thread->t_ctl_waitq);
1472
1473         watchdog = lc_watchdog_add(max_t(int, obd_timeout, AT_OFF ? 0 :
1474                                    at_get(&svc->srv_at_estimate)) *
1475                                    svc->srv_watchdog_factor, NULL, NULL);
1476
1477         spin_lock(&svc->srv_lock);
1478         svc->srv_threads_running++;
1479         list_add(&rs->rs_list, &svc->srv_free_rs_list);
1480         spin_unlock(&svc->srv_lock);
1481         cfs_waitq_signal(&svc->srv_free_rs_waitq);
1482
1483         CDEBUG(D_NET, "service thread %d (#%d)started\n", thread->t_id,
1484               svc->srv_threads_running);
1485
1486         /* XXX maintain a list of all managed devices: insert here */
1487
1488         while ((thread->t_flags & SVC_STOPPING) == 0 ||
1489                svc->srv_n_difficult_replies != 0) {
1490                 /* Don't exit while there are replies to be handled */
1491                 struct l_wait_info lwi = LWI_TIMEOUT(svc->srv_rqbd_timeout,
1492                                                      ptlrpc_retry_rqbds, svc);
1493
1494                 lc_watchdog_disable(watchdog);
1495
1496                 cond_resched();
1497
1498                 l_wait_event_exclusive (svc->srv_waitq,
1499                               ((thread->t_flags & SVC_STOPPING) != 0 &&
1500                                svc->srv_n_difficult_replies == 0) ||
1501                               (!list_empty(&svc->srv_idle_rqbds) &&
1502                                svc->srv_rqbd_timeout == 0) ||
1503                               !list_empty(&svc->srv_req_in_queue) ||
1504                               !list_empty(&svc->srv_reply_queue) ||
1505                               (!list_empty(&svc->srv_request_queue) &&
1506                                (svc->srv_n_active_reqs <
1507                                 (svc->srv_threads_running - 1))) ||
1508                               svc->srv_at_check,
1509                               &lwi);
1510
1511                 lc_watchdog_touch_ms(watchdog, max_t(int, obd_timeout,
1512                                      AT_OFF ? 0 :
1513                                      at_get(&svc->srv_at_estimate)) *
1514                                      svc->srv_watchdog_factor);
1515
1516                 ptlrpc_check_rqbd_pool(svc);
1517
1518                 if ((svc->srv_threads_started < svc->srv_threads_max) &&
1519                     (svc->srv_n_active_reqs >= (svc->srv_threads_started - 1))){
1520                         /* Ignore return code - we tried... */
1521                         ptlrpc_start_thread(dev, svc);
1522                 }
1523
1524                 if (!list_empty(&svc->srv_reply_queue))
1525                         ptlrpc_server_handle_reply(svc);
1526
1527                 if (!list_empty(&svc->srv_req_in_queue)) {
1528                         /* Process all incoming reqs before handling any */
1529                         ptlrpc_server_handle_req_in(svc);
1530                         /* but limit ourselves in case of flood */
1531                         if (counter++ < 1000)
1532                                 continue;
1533                         counter = 0;
1534                 }
1535
1536                 if (svc->srv_at_check)
1537                         ptlrpc_at_check_timed(svc);
1538
1539                 /* don't handle requests in the last thread */
1540                 if (!list_empty (&svc->srv_request_queue) &&
1541                     (svc->srv_n_active_reqs < (svc->srv_threads_running - 1))) {
1542                         lu_context_enter(&env.le_ctx);
1543                         ptlrpc_server_handle_request(svc, thread);
1544                         lu_context_exit(&env.le_ctx);
1545                 }
1546
1547                 if (!list_empty(&svc->srv_idle_rqbds) &&
1548                     ptlrpc_server_post_idle_rqbds(svc) < 0) {
1549                         /* I just failed to repost request buffers.  Wait
1550                          * for a timeout (unless something else happens)
1551                          * before I try again */
1552                         svc->srv_rqbd_timeout = cfs_time_seconds(1)/10;
1553                         CDEBUG(D_RPCTRACE,"Posted buffers: %d\n",
1554                                svc->srv_nrqbd_receiving);
1555                 }
1556         }
1557
1558         lc_watchdog_delete(watchdog);
1559
1560 out_srv_fini:
1561         /*
1562          * deconstruct service specific state created by ptlrpc_start_thread()
1563          */
1564         if (svc->srv_done != NULL)
1565                 svc->srv_done(thread);
1566
1567         lu_context_fini(&env.le_ctx);
1568 out:
1569         CDEBUG(D_NET, "service thread %d exiting: rc %d\n", thread->t_id, rc);
1570
1571         spin_lock(&svc->srv_lock);
1572         svc->srv_threads_running--; /* must know immediately */
1573         thread->t_id = rc;
1574         thread->t_flags = SVC_STOPPED;
1575
1576         cfs_waitq_signal(&thread->t_ctl_waitq);
1577         spin_unlock(&svc->srv_lock);
1578
1579         return rc;
1580 }
1581
1582 static void ptlrpc_stop_thread(struct ptlrpc_service *svc,
1583                                struct ptlrpc_thread *thread)
1584 {
1585         struct l_wait_info lwi = { 0 };
1586
1587         spin_lock(&svc->srv_lock);
1588         thread->t_flags = SVC_STOPPING;
1589         spin_unlock(&svc->srv_lock);
1590
1591         cfs_waitq_broadcast(&svc->srv_waitq);
1592         l_wait_event(thread->t_ctl_waitq, (thread->t_flags & SVC_STOPPED),
1593                      &lwi);
1594
1595         spin_lock(&svc->srv_lock);
1596         list_del(&thread->t_link);
1597         spin_unlock(&svc->srv_lock);
1598
1599         OBD_FREE_PTR(thread);
1600 }
1601
1602 void ptlrpc_stop_all_threads(struct ptlrpc_service *svc)
1603 {
1604         struct ptlrpc_thread *thread;
1605
1606         spin_lock(&svc->srv_lock);
1607         while (!list_empty(&svc->srv_threads)) {
1608                 thread = list_entry(svc->srv_threads.next,
1609                                     struct ptlrpc_thread, t_link);
1610
1611                 spin_unlock(&svc->srv_lock);
1612                 ptlrpc_stop_thread(svc, thread);
1613                 spin_lock(&svc->srv_lock);
1614         }
1615
1616         spin_unlock(&svc->srv_lock);
1617 }
1618
1619 int ptlrpc_start_threads(struct obd_device *dev, struct ptlrpc_service *svc)
1620 {
1621         int i, rc = 0;
1622         ENTRY;
1623
1624         /* We require 2 threads min - see note in
1625            ptlrpc_server_handle_request */
1626         LASSERT(svc->srv_threads_min >= 2);
1627         for (i = 0; i < svc->srv_threads_min; i++) {
1628                 rc = ptlrpc_start_thread(dev, svc);
1629                 /* We have enough threads, don't start more.  b=15759 */
1630                 if (rc == -EMFILE)
1631                         break;
1632                 if (rc) {
1633                         CERROR("cannot start %s thread #%d: rc %d\n",
1634                                svc->srv_thread_name, i, rc);
1635                         ptlrpc_stop_all_threads(svc);
1636                 }
1637         }
1638         RETURN(rc);
1639 }
1640
1641 int ptlrpc_start_thread(struct obd_device *dev, struct ptlrpc_service *svc)
1642 {
1643         struct l_wait_info lwi = { 0 };
1644         struct ptlrpc_svc_data d;
1645         struct ptlrpc_thread *thread;
1646         char name[32];
1647         int id, rc;
1648         ENTRY;
1649
1650         CDEBUG(D_RPCTRACE, "%s started %d min %d max %d running %d\n",
1651                svc->srv_name, svc->srv_threads_started, svc->srv_threads_min,
1652                svc->srv_threads_max, svc->srv_threads_running);
1653         if (unlikely(svc->srv_threads_started >= svc->srv_threads_max) ||
1654             (OBD_FAIL_CHECK(OBD_FAIL_TGT_TOOMANY_THREADS) &&
1655              svc->srv_threads_started == svc->srv_threads_min - 1))
1656                 RETURN(-EMFILE);
1657
1658         OBD_ALLOC_PTR(thread);
1659         if (thread == NULL)
1660                 RETURN(-ENOMEM);
1661         cfs_waitq_init(&thread->t_ctl_waitq);
1662
1663         spin_lock(&svc->srv_lock);
1664         if (svc->srv_threads_started >= svc->srv_threads_max) {
1665                 spin_unlock(&svc->srv_lock);
1666                 OBD_FREE_PTR(thread);
1667                 RETURN(-EMFILE);
1668         }
1669         list_add(&thread->t_link, &svc->srv_threads);
1670         id = svc->srv_threads_started++;
1671         spin_unlock(&svc->srv_lock);
1672
1673         thread->t_id = id;
1674         sprintf(name, "%s_%02d", svc->srv_thread_name, id);
1675         d.dev = dev;
1676         d.svc = svc;
1677         d.name = name;
1678         d.thread = thread;
1679
1680         CDEBUG(D_RPCTRACE, "starting thread '%s'\n", name);
1681
1682           /* CLONE_VM and CLONE_FILES just avoid a needless copy, because we
1683          * just drop the VM and FILES in ptlrpc_daemonize() right away.
1684          */
1685         rc = cfs_kernel_thread(ptlrpc_main, &d, CLONE_VM | CLONE_FILES);
1686         if (rc < 0) {
1687                 CERROR("cannot start thread '%s': rc %d\n", name, rc);
1688
1689                 spin_lock(&svc->srv_lock);
1690                 list_del(&thread->t_link);
1691                 --svc->srv_threads_started;
1692                 spin_unlock(&svc->srv_lock);
1693
1694                 OBD_FREE(thread, sizeof(*thread));
1695                 RETURN(rc);
1696         }
1697         l_wait_event(thread->t_ctl_waitq,
1698                      thread->t_flags & (SVC_RUNNING | SVC_STOPPED), &lwi);
1699
1700         rc = (thread->t_flags & SVC_STOPPED) ? thread->t_id : 0;
1701         RETURN(rc);
1702 }
1703 #endif
1704
1705 int ptlrpc_unregister_service(struct ptlrpc_service *service)
1706 {
1707         int                   rc;
1708         struct l_wait_info    lwi;
1709         struct list_head     *tmp;
1710         struct ptlrpc_reply_state *rs, *t;
1711
1712         cfs_timer_disarm(&service->srv_at_timer);
1713
1714         ptlrpc_stop_all_threads(service);
1715         LASSERT(list_empty(&service->srv_threads));
1716
1717         spin_lock (&ptlrpc_all_services_lock);
1718         list_del_init (&service->srv_list);
1719         spin_unlock (&ptlrpc_all_services_lock);
1720
1721         ptlrpc_lprocfs_unregister_service(service);
1722
1723         /* All history will be culled when the next request buffer is
1724          * freed */
1725         service->srv_max_history_rqbds = 0;
1726
1727         CDEBUG(D_NET, "%s: tearing down\n", service->srv_name);
1728
1729         rc = LNetClearLazyPortal(service->srv_req_portal);
1730         LASSERT (rc == 0);
1731
1732         /* Unlink all the request buffers.  This forces a 'final' event with
1733          * its 'unlink' flag set for each posted rqbd */
1734         list_for_each(tmp, &service->srv_active_rqbds) {
1735                 struct ptlrpc_request_buffer_desc *rqbd =
1736                         list_entry(tmp, struct ptlrpc_request_buffer_desc,
1737                                    rqbd_list);
1738
1739                 rc = LNetMDUnlink(rqbd->rqbd_md_h);
1740                 LASSERT (rc == 0 || rc == -ENOENT);
1741         }
1742
1743         /* Wait for the network to release any buffers it's currently
1744          * filling */
1745         for (;;) {
1746                 spin_lock(&service->srv_lock);
1747                 rc = service->srv_nrqbd_receiving;
1748                 spin_unlock(&service->srv_lock);
1749
1750                 if (rc == 0)
1751                         break;
1752
1753                 /* Network access will complete in finite time but the HUGE
1754                  * timeout lets us CWARN for visibility of sluggish NALs */
1755                 lwi = LWI_TIMEOUT(cfs_time_seconds(LONG_UNLINK), NULL, NULL);
1756                 rc = l_wait_event(service->srv_waitq,
1757                                   service->srv_nrqbd_receiving == 0,
1758                                   &lwi);
1759                 if (rc == -ETIMEDOUT)
1760                         CWARN("Service %s waiting for request buffers\n",
1761                               service->srv_name);
1762         }
1763
1764         /* schedule all outstanding replies to terminate them */
1765         spin_lock(&service->srv_lock);
1766         while (!list_empty(&service->srv_active_replies)) {
1767                 struct ptlrpc_reply_state *rs =
1768                         list_entry(service->srv_active_replies.next,
1769                                    struct ptlrpc_reply_state, rs_list);
1770                 ptlrpc_schedule_difficult_reply(rs);
1771         }
1772         spin_unlock(&service->srv_lock);
1773
1774         /* purge the request queue.  NB No new replies (rqbds all unlinked)
1775          * and no service threads, so I'm the only thread noodling the
1776          * request queue now */
1777         while (!list_empty(&service->srv_req_in_queue)) {
1778                 struct ptlrpc_request *req =
1779                         list_entry(service->srv_req_in_queue.next,
1780                                    struct ptlrpc_request,
1781                                    rq_list);
1782
1783                 list_del(&req->rq_list);
1784                 service->srv_n_queued_reqs--;
1785                 service->srv_n_active_reqs++;
1786                 ptlrpc_server_finish_request(req);
1787         }
1788         while (!list_empty(&service->srv_request_queue)) {
1789                 struct ptlrpc_request *req =
1790                         list_entry(service->srv_request_queue.next,
1791                                    struct ptlrpc_request,
1792                                    rq_list);
1793
1794                 list_del(&req->rq_list);
1795                 service->srv_n_queued_reqs--;
1796                 service->srv_n_active_reqs++;
1797
1798                 ptlrpc_server_finish_request(req);
1799         }
1800         LASSERT(service->srv_n_queued_reqs == 0);
1801         LASSERT(service->srv_n_active_reqs == 0);
1802         LASSERT(service->srv_n_history_rqbds == 0);
1803         LASSERT(list_empty(&service->srv_active_rqbds));
1804
1805         /* Now free all the request buffers since nothing references them
1806          * any more... */
1807         while (!list_empty(&service->srv_idle_rqbds)) {
1808                 struct ptlrpc_request_buffer_desc *rqbd =
1809                         list_entry(service->srv_idle_rqbds.next,
1810                                    struct ptlrpc_request_buffer_desc,
1811                                    rqbd_list);
1812
1813                 ptlrpc_free_rqbd(rqbd);
1814         }
1815
1816         /* wait for all outstanding replies to complete (they were
1817          * scheduled having been flagged to abort above) */
1818         while (atomic_read(&service->srv_outstanding_replies) != 0) {
1819                 struct l_wait_info lwi = LWI_TIMEOUT(cfs_time_seconds(10), NULL, NULL);
1820
1821                 rc = l_wait_event(service->srv_waitq,
1822                                   !list_empty(&service->srv_reply_queue), &lwi);
1823                 LASSERT(rc == 0 || rc == -ETIMEDOUT);
1824
1825                 if (rc == 0) {
1826                         ptlrpc_server_handle_reply(service);
1827                         continue;
1828                 }
1829                 CWARN("Unexpectedly long timeout %p\n", service);
1830         }
1831
1832         list_for_each_entry_safe(rs, t, &service->srv_free_rs_list, rs_list) {
1833                 list_del(&rs->rs_list);
1834                 OBD_FREE(rs, service->srv_max_reply_size);
1835         }
1836
1837         /* In case somebody rearmed this in the meantime */
1838         cfs_timer_disarm(&service->srv_at_timer);
1839
1840         OBD_FREE_PTR(service);
1841         return 0;
1842 }
1843
1844 /* Returns 0 if the service is healthy.
1845  *
1846  * Right now, it just checks to make sure that requests aren't languishing
1847  * in the queue.  We'll use this health check to govern whether a node needs
1848  * to be shot, so it's intentionally non-aggressive. */
1849 int ptlrpc_service_health_check(struct ptlrpc_service *svc)
1850 {
1851         struct ptlrpc_request *request;
1852         struct timeval         right_now;
1853         long                   timediff;
1854
1855         if (svc == NULL)
1856                 return 0;
1857
1858         do_gettimeofday(&right_now);
1859
1860         spin_lock(&svc->srv_lock);
1861         if (list_empty(&svc->srv_request_queue)) {
1862                 spin_unlock(&svc->srv_lock);
1863                 return 0;
1864         }
1865
1866         /* How long has the next entry been waiting? */
1867         request = list_entry(svc->srv_request_queue.next,
1868                              struct ptlrpc_request, rq_list);
1869         timediff = cfs_timeval_sub(&right_now, &request->rq_arrival_time, NULL);
1870         spin_unlock(&svc->srv_lock);
1871
1872         if ((timediff / ONE_MILLION) > (AT_OFF ? obd_timeout * 3/2 :
1873                                         at_max)) {
1874                 CERROR("%s: unhealthy - request has been waiting %lds\n",
1875                        svc->srv_name, timediff / ONE_MILLION);
1876                 return (-1);
1877         }
1878
1879         return 0;
1880 }