Whamcloud - gitweb
LU-6245 libcfs: remove libcfsutil.h
[fs/lustre-release.git] / lnet / selftest / console.c
1 /*
2  * GPL HEADER START
3  *
4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 only,
8  * as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13  * General Public License version 2 for more details (a copy is included
14  * in the LICENSE file that accompanied this code).
15  *
16  * You should have received a copy of the GNU General Public License
17  * version 2 along with this program; If not, see
18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
19  *
20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21  * CA 95054 USA or visit www.sun.com if you need additional information or
22  * have any questions.
23  *
24  * GPL HEADER END
25  */
26 /*
27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
28  * Use is subject to license terms.
29  *
30  * Copyright (c) 2012, 2014, Intel Corporation.
31  */
32 /*
33  * This file is part of Lustre, http://www.lustre.org/
34  * Lustre is a trademark of Sun Microsystems, Inc.
35  *
36  * lnet/selftest/conctl.c
37  *
38  * Infrastructure of LST console
39  *
40  * Author: Liang Zhen <liangzhen@clusterfs.com>
41  */
42
43
44 #include <libcfs/libcfs.h>
45 #include <lnet/lib-lnet.h>
46 #include "console.h"
47 #include "conrpc.h"
48
49 #define LST_NODE_STATE_COUNTER(nd, p)                   \
50 do {                                                    \
51         if ((nd)->nd_state == LST_NODE_ACTIVE)          \
52                 (p)->nle_nactive ++;                    \
53         else if ((nd)->nd_state == LST_NODE_BUSY)       \
54                 (p)->nle_nbusy ++;                      \
55         else if ((nd)->nd_state == LST_NODE_DOWN)       \
56                 (p)->nle_ndown ++;                      \
57         else                                            \
58                 (p)->nle_nunknown ++;                   \
59         (p)->nle_nnode ++;                              \
60 } while (0)
61
62 lstcon_session_t        console_session;
63
64 static void
65 lstcon_node_get(lstcon_node_t *nd)
66 {
67         LASSERT (nd->nd_ref >= 1);
68
69         nd->nd_ref++;
70 }
71
72 static int
73 lstcon_node_find(lnet_process_id_t id, lstcon_node_t **ndpp, int create)
74 {
75         lstcon_ndlink_t *ndl;
76         unsigned int     idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE;
77
78         LASSERT(id.nid != LNET_NID_ANY);
79
80         list_for_each_entry(ndl, &console_session.ses_ndl_hash[idx],
81                             ndl_hlink) {
82                 if (ndl->ndl_node->nd_id.nid != id.nid ||
83                     ndl->ndl_node->nd_id.pid != id.pid)
84                         continue;
85
86                 lstcon_node_get(ndl->ndl_node);
87                 *ndpp = ndl->ndl_node;
88                 return 0;
89         }
90
91         if (!create)
92                 return -ENOENT;
93
94         LIBCFS_ALLOC(*ndpp, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
95         if (*ndpp == NULL)
96                 return -ENOMEM;
97
98         ndl = (lstcon_ndlink_t *)(*ndpp + 1);
99
100         ndl->ndl_node = *ndpp;
101
102         ndl->ndl_node->nd_ref   = 1;
103         ndl->ndl_node->nd_id    = id;
104         ndl->ndl_node->nd_stamp = cfs_time_current();
105         ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
106         ndl->ndl_node->nd_timeout = 0;
107         memset(&ndl->ndl_node->nd_ping, 0, sizeof(lstcon_rpc_t));
108
109         /* queued in global hash & list, no refcount is taken by
110          * global hash & list, if caller release his refcount,
111          * node will be released */
112         list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]);
113         list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list);
114
115         return 0;
116 }
117
118 static void
119 lstcon_node_put(lstcon_node_t *nd)
120 {
121         lstcon_ndlink_t *ndl;
122
123         LASSERT(nd->nd_ref > 0);
124
125         if (--nd->nd_ref > 0)
126                 return;
127
128         ndl = (lstcon_ndlink_t *)(nd + 1);
129
130         LASSERT(!list_empty(&ndl->ndl_link));
131         LASSERT(!list_empty(&ndl->ndl_hlink));
132
133         /* remove from session */
134         list_del(&ndl->ndl_link);
135         list_del(&ndl->ndl_hlink);
136
137         LIBCFS_FREE(nd, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
138 }
139
140 static int
141 lstcon_ndlink_find(struct list_head *hash,
142                    lnet_process_id_t id, lstcon_ndlink_t **ndlpp, int create)
143 {
144         unsigned int     idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
145         lstcon_ndlink_t *ndl;
146         lstcon_node_t   *nd;
147         int              rc;
148
149         if (id.nid == LNET_NID_ANY)
150                 return -EINVAL;
151
152         /* search in hash */
153         list_for_each_entry(ndl, &hash[idx], ndl_hlink) {
154                 if (ndl->ndl_node->nd_id.nid != id.nid ||
155                     ndl->ndl_node->nd_id.pid != id.pid)
156                         continue;
157
158                 *ndlpp = ndl;
159                 return 0;
160         }
161
162         if (create == 0)
163                 return -ENOENT;
164
165         /* find or create in session hash */
166         rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0);
167         if (rc != 0)
168                 return rc;
169
170         LIBCFS_ALLOC(ndl, sizeof(lstcon_ndlink_t));
171         if (ndl == NULL) {
172                 lstcon_node_put(nd);
173                 return -ENOMEM;
174         }
175
176         *ndlpp = ndl;
177
178         ndl->ndl_node = nd;
179         INIT_LIST_HEAD(&ndl->ndl_link);
180         list_add_tail(&ndl->ndl_hlink, &hash[idx]);
181
182         return 0;
183 }
184
185 static void
186 lstcon_ndlink_release(lstcon_ndlink_t *ndl)
187 {
188         LASSERT(list_empty(&ndl->ndl_link));
189         LASSERT(!list_empty(&ndl->ndl_hlink));
190
191         list_del(&ndl->ndl_hlink); /* delete from hash */
192         lstcon_node_put(ndl->ndl_node);
193
194         LIBCFS_FREE(ndl, sizeof(*ndl));
195 }
196
197 static int
198 lstcon_group_alloc(char *name, lstcon_group_t **grpp)
199 {
200         lstcon_group_t *grp;
201         int             i;
202
203         LIBCFS_ALLOC(grp, offsetof(lstcon_group_t,
204                                    grp_ndl_hash[LST_NODE_HASHSIZE]));
205         if (grp == NULL)
206                 return -ENOMEM;
207
208         memset(grp, 0, offsetof(lstcon_group_t,
209                                 grp_ndl_hash[LST_NODE_HASHSIZE]));
210
211         grp->grp_ref = 1;
212         if (name != NULL) {
213                 if (strlen(name) > sizeof(grp->grp_name)-1) {
214                         LIBCFS_FREE(grp, offsetof(lstcon_group_t,
215                                           grp_ndl_hash[LST_NODE_HASHSIZE]));
216                         return -E2BIG;
217                 }
218                 strncpy(grp->grp_name, name, sizeof(grp->grp_name));
219         }
220
221         INIT_LIST_HEAD(&grp->grp_link);
222         INIT_LIST_HEAD(&grp->grp_ndl_list);
223         INIT_LIST_HEAD(&grp->grp_trans_list);
224
225         for (i = 0; i < LST_NODE_HASHSIZE; i++)
226                 INIT_LIST_HEAD(&grp->grp_ndl_hash[i]);
227
228         *grpp = grp;
229
230         return 0;
231 }
232
233 static void
234 lstcon_group_addref(lstcon_group_t *grp)
235 {
236         grp->grp_ref++;
237 }
238
239 static void lstcon_group_ndlink_release(lstcon_group_t *, lstcon_ndlink_t *);
240
241 static void
242 lstcon_group_drain(lstcon_group_t *grp, int keep)
243 {
244         lstcon_ndlink_t *ndl;
245         lstcon_ndlink_t *tmp;
246
247         list_for_each_entry_safe(ndl, tmp, &grp->grp_ndl_list, ndl_link) {
248                 if ((ndl->ndl_node->nd_state & keep) == 0)
249                         lstcon_group_ndlink_release(grp, ndl);
250         }
251 }
252
253 static void
254 lstcon_group_decref(lstcon_group_t *grp)
255 {
256         int i;
257
258         if (--grp->grp_ref > 0)
259                 return;
260
261         if (!list_empty(&grp->grp_link))
262                 list_del(&grp->grp_link);
263
264         lstcon_group_drain(grp, 0);
265
266         for (i = 0; i < LST_NODE_HASHSIZE; i++)
267                 LASSERT(list_empty(&grp->grp_ndl_hash[i]));
268
269         LIBCFS_FREE(grp, offsetof(lstcon_group_t,
270                                   grp_ndl_hash[LST_NODE_HASHSIZE]));
271 }
272
273 static int
274 lstcon_group_find(const char *name, lstcon_group_t **grpp)
275 {
276         lstcon_group_t *grp;
277
278         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
279                 if (strncmp(grp->grp_name, name, LST_NAME_SIZE) != 0)
280                         continue;
281
282                 lstcon_group_addref(grp);  /* +1 ref for caller */
283                 *grpp = grp;
284                 return 0;
285         }
286
287         return -ENOENT;
288 }
289
290 static void
291 lstcon_group_put(lstcon_group_t *grp)
292 {
293         lstcon_group_decref(grp);
294 }
295
296 static int
297 lstcon_group_ndlink_find(lstcon_group_t *grp, lnet_process_id_t id,
298                          lstcon_ndlink_t **ndlpp, int create)
299 {
300         int rc;
301
302         rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create);
303         if (rc != 0)
304                 return rc;
305
306         if (!list_empty(&(*ndlpp)->ndl_link))
307                 return 0;
308
309         list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list);
310         grp->grp_nnode++;
311
312         return 0;
313 }
314
315 static void
316 lstcon_group_ndlink_release(lstcon_group_t *grp, lstcon_ndlink_t *ndl)
317 {
318         list_del_init(&ndl->ndl_link);
319         lstcon_ndlink_release(ndl);
320         grp->grp_nnode--;
321 }
322
323 static void
324 lstcon_group_ndlink_move(lstcon_group_t *old,
325                          lstcon_group_t *new, lstcon_ndlink_t *ndl)
326 {
327         unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) %
328                                         LST_NODE_HASHSIZE;
329
330         list_del(&ndl->ndl_hlink);
331         list_del(&ndl->ndl_link);
332         old->grp_nnode--;
333
334         list_add_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]);
335         list_add_tail(&ndl->ndl_link, &new->grp_ndl_list);
336         new->grp_nnode++;
337
338         return;
339 }
340
341 static void
342 lstcon_group_move(lstcon_group_t *old, lstcon_group_t *new)
343 {
344         lstcon_ndlink_t *ndl;
345
346         while (!list_empty(&old->grp_ndl_list)) {
347                 ndl = list_entry(old->grp_ndl_list.next,
348                                  lstcon_ndlink_t, ndl_link);
349                 lstcon_group_ndlink_move(old, new, ndl);
350         }
351 }
352
353 static int
354 lstcon_sesrpc_condition(int transop, lstcon_node_t *nd, void *arg)
355 {
356         lstcon_group_t *grp = (lstcon_group_t *)arg;
357
358         switch (transop) {
359         case LST_TRANS_SESNEW:
360                 if (nd->nd_state == LST_NODE_ACTIVE)
361                         return 0;
362                 break;
363
364         case LST_TRANS_SESEND:
365                 if (nd->nd_state != LST_NODE_ACTIVE)
366                         return 0;
367
368                 if (grp != NULL && nd->nd_ref > 1)
369                         return 0;
370                 break;
371
372         case LST_TRANS_SESQRY:
373                 break;
374
375         default:
376                 LBUG();
377         }
378
379         return 1;
380 }
381
382 static int
383 lstcon_sesrpc_readent(int transop, srpc_msg_t *msg,
384                       lstcon_rpc_ent_t __user *ent_up)
385 {
386         srpc_debug_reply_t *rep;
387
388         switch (transop) {
389         case LST_TRANS_SESNEW:
390         case LST_TRANS_SESEND:
391                 return 0;
392
393         case LST_TRANS_SESQRY:
394                 rep = &msg->msg_body.dbg_reply;
395
396                 if (copy_to_user(&ent_up->rpe_priv[0],
397                                      &rep->dbg_timeout, sizeof(int)) ||
398                     copy_to_user(&ent_up->rpe_payload[0],
399                                      &rep->dbg_name, LST_NAME_SIZE))
400                         return -EFAULT;
401
402                 return 0;
403
404         default:
405                 LBUG();
406         }
407
408         return 0;
409 }
410
411 static int
412 lstcon_group_nodes_add(lstcon_group_t *grp,
413                        int count, lnet_process_id_t __user *ids_up,
414                        unsigned *featp, struct list_head __user *result_up)
415 {
416         lstcon_rpc_trans_t      *trans;
417         lstcon_ndlink_t         *ndl;
418         lstcon_group_t          *tmp;
419         lnet_process_id_t        id;
420         int                      i;
421         int                      rc;
422
423         rc = lstcon_group_alloc(NULL, &tmp);
424         if (rc != 0) {
425                 CERROR("Out of memory\n");
426                 return -ENOMEM;
427         }
428
429         for (i = 0 ; i < count; i++) {
430                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
431                         rc = -EFAULT;
432                         break;
433                 }
434
435                 /* skip if it's in this group already */
436                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 0);
437                 if (rc == 0)
438                         continue;
439
440                 /* add to tmp group */
441                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1);
442                 if (rc != 0) {
443                         CERROR("Can't create ndlink, out of memory\n");
444                         break;
445                 }
446         }
447
448         if (rc != 0) {
449                 lstcon_group_put(tmp);
450                 return rc;
451         }
452
453         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
454                                      &tmp->grp_trans_list, LST_TRANS_SESNEW,
455                                      tmp, lstcon_sesrpc_condition, &trans);
456         if (rc != 0) {
457                 CERROR("Can't create transaction: %d\n", rc);
458                 lstcon_group_put(tmp);
459                 return rc;
460         }
461
462         /* post all RPCs */
463         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
464
465         rc = lstcon_rpc_trans_interpreter(trans, result_up,
466                                           lstcon_sesrpc_readent);
467         *featp = trans->tas_features;
468
469         /* destroy all RPGs */
470         lstcon_rpc_trans_destroy(trans);
471
472         lstcon_group_move(tmp, grp);
473         lstcon_group_put(tmp);
474
475         return rc;
476 }
477
478 static int
479 lstcon_group_nodes_remove(lstcon_group_t *grp,
480                           int count, lnet_process_id_t __user *ids_up,
481                           struct list_head __user *result_up)
482 {
483         lstcon_rpc_trans_t     *trans;
484         lstcon_ndlink_t        *ndl;
485         lstcon_group_t         *tmp;
486         lnet_process_id_t       id;
487         int                     rc;
488         int                     i;
489
490         /* End session and remove node from the group */
491
492         rc = lstcon_group_alloc(NULL, &tmp);
493         if (rc != 0) {
494                 CERROR("Out of memory\n");
495                 return -ENOMEM;
496         }
497
498         for (i = 0; i < count; i++) {
499                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
500                         rc = -EFAULT;
501                         goto error;
502                 }
503
504                 /* move node to tmp group */
505                 if (lstcon_group_ndlink_find(grp, id, &ndl, 0) == 0)
506                         lstcon_group_ndlink_move(grp, tmp, ndl);
507         }
508
509         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
510                                      &tmp->grp_trans_list, LST_TRANS_SESEND,
511                                      tmp, lstcon_sesrpc_condition, &trans);
512         if (rc != 0) {
513                 CERROR("Can't create transaction: %d\n", rc);
514                 goto error;
515         }
516
517         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
518
519         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
520
521         lstcon_rpc_trans_destroy(trans);
522         /* release nodes anyway, because we can't rollback status */
523         lstcon_group_put(tmp);
524
525         return rc;
526 error:
527         lstcon_group_move(tmp, grp);
528         lstcon_group_put(tmp);
529
530         return rc;
531 }
532
533 int
534 lstcon_group_add(char *name)
535 {
536         lstcon_group_t *grp;
537         int             rc;
538
539         rc = (lstcon_group_find(name, &grp) == 0)? -EEXIST: 0;
540         if (rc != 0) {
541                 /* find a group with same name */
542                 lstcon_group_put(grp);
543                 return rc;
544         }
545
546         rc = lstcon_group_alloc(name, &grp);
547         if (rc != 0) {
548                 CERROR("Can't allocate descriptor for group %s\n", name);
549                 return -ENOMEM;
550         }
551
552         list_add_tail(&grp->grp_link, &console_session.ses_grp_list);
553
554         return rc;
555 }
556
557 int
558 lstcon_nodes_add(char *name, int count, lnet_process_id_t __user *ids_up,
559                  unsigned *featp, struct list_head __user *result_up)
560 {
561         lstcon_group_t         *grp;
562         int                     rc;
563
564         LASSERT (count > 0);
565         LASSERT (ids_up != NULL);
566
567         rc = lstcon_group_find(name, &grp);
568         if (rc != 0) {
569                 CDEBUG(D_NET, "Can't find group %s\n", name);
570                 return rc;
571         }
572
573         if (grp->grp_ref > 2) {
574                 /* referred by other threads or test */
575                 CDEBUG(D_NET, "Group %s is busy\n", name);
576                 lstcon_group_put(grp);
577
578                 return -EBUSY;
579         }
580
581         rc = lstcon_group_nodes_add(grp, count, ids_up, featp, result_up);
582
583         lstcon_group_put(grp);
584
585         return rc;
586 }
587
588 int
589 lstcon_group_del(char *name)
590 {
591         lstcon_rpc_trans_t *trans;
592         lstcon_group_t     *grp;
593         int                 rc;
594
595         rc = lstcon_group_find(name, &grp);
596         if (rc != 0) {
597                 CDEBUG(D_NET, "Can't find group: %s\n", name);
598                 return rc;
599         }
600
601         if (grp->grp_ref > 2) {
602                 /* referred by others threads or test */
603                 CDEBUG(D_NET, "Group %s is busy\n", name);
604                 lstcon_group_put(grp);
605                 return -EBUSY;
606         }
607
608         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
609                                      &grp->grp_trans_list, LST_TRANS_SESEND,
610                                      grp, lstcon_sesrpc_condition, &trans);
611         if (rc != 0) {
612                 CERROR("Can't create transaction: %d\n", rc);
613                 lstcon_group_put(grp);
614                 return rc;
615         }
616
617         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
618
619         lstcon_rpc_trans_destroy(trans);
620
621         lstcon_group_put(grp);
622         /* -ref for session, it's destroyed,
623          * status can't be rolled back, destroy group anway */
624         lstcon_group_put(grp);
625
626         return rc;
627 }
628
629 int
630 lstcon_group_clean(char *name, int args)
631 {
632         lstcon_group_t *grp = NULL;
633         int             rc;
634
635         rc = lstcon_group_find(name, &grp);
636         if (rc != 0) {
637                 CDEBUG(D_NET, "Can't find group %s\n", name);
638                 return rc;
639         }
640
641         if (grp->grp_ref > 2) {
642                 /* referred by test */
643                 CDEBUG(D_NET, "Group %s is busy\n", name);
644                 lstcon_group_put(grp);
645                 return -EBUSY;
646         }
647
648         args = (LST_NODE_ACTIVE | LST_NODE_BUSY |
649                 LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args;
650
651         lstcon_group_drain(grp, args);
652
653         lstcon_group_put(grp);
654         /* release empty group */
655         if (list_empty(&grp->grp_ndl_list))
656                 lstcon_group_put(grp);
657
658         return 0;
659 }
660
661 int
662 lstcon_nodes_remove(char *name, int count,
663                     lnet_process_id_t __user *ids_up,
664                     struct list_head __user *result_up)
665 {
666         lstcon_group_t *grp = NULL;
667         int             rc;
668
669         rc = lstcon_group_find(name, &grp);
670         if (rc != 0) {
671                 CDEBUG(D_NET, "Can't find group: %s\n", name);
672                 return rc;
673         }
674
675         if (grp->grp_ref > 2) {
676                 /* referred by test */
677                 CDEBUG(D_NET, "Group %s is busy\n", name);
678                 lstcon_group_put(grp);
679                 return -EBUSY;
680         }
681
682         rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up);
683
684         lstcon_group_put(grp);
685         /* release empty group */
686         if (list_empty(&grp->grp_ndl_list))
687                 lstcon_group_put(grp);
688
689         return rc;
690 }
691
692 int
693 lstcon_group_refresh(char *name, struct list_head __user *result_up)
694 {
695         lstcon_rpc_trans_t      *trans;
696         lstcon_group_t          *grp;
697         int                      rc;
698
699         rc = lstcon_group_find(name, &grp);
700         if (rc != 0) {
701                 CDEBUG(D_NET, "Can't find group: %s\n", name);
702                 return rc;
703         }
704
705         if (grp->grp_ref > 2) {
706                 /* referred by test */
707                 CDEBUG(D_NET, "Group %s is busy\n", name);
708                 lstcon_group_put(grp);
709                 return -EBUSY;
710         }
711
712         /* re-invite all inactive nodes int the group */
713         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
714                                      &grp->grp_trans_list, LST_TRANS_SESNEW,
715                                      grp, lstcon_sesrpc_condition, &trans);
716         if (rc != 0) {
717                 /* local error, return */
718                 CDEBUG(D_NET, "Can't create transaction: %d\n", rc);
719                 lstcon_group_put(grp);
720                 return rc;
721         }
722
723         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
724
725         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
726
727         lstcon_rpc_trans_destroy(trans);
728         /* -ref for me */
729         lstcon_group_put(grp);
730
731         return rc;
732 }
733
734 int
735 lstcon_group_list(int index, int len, char __user *name_up)
736 {
737         lstcon_group_t *grp;
738
739         LASSERT(index >= 0);
740         LASSERT(name_up != NULL);
741
742         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
743                 if (index-- == 0) {
744                         return copy_to_user(name_up, grp->grp_name, len) ?
745                                             -EFAULT : 0;
746                 }
747         }
748
749         return -ENOENT;
750 }
751
752 static int
753 lstcon_nodes_getent(struct list_head *head, int *index_p,
754                     int *count_p, lstcon_node_ent_t __user *dents_up)
755 {
756         lstcon_ndlink_t  *ndl;
757         lstcon_node_t    *nd;
758         int               count = 0;
759         int               index = 0;
760
761         LASSERT(index_p != NULL && count_p != NULL);
762         LASSERT(dents_up != NULL);
763         LASSERT(*index_p >= 0);
764         LASSERT(*count_p > 0);
765
766         list_for_each_entry(ndl, head, ndl_link) {
767                 if (index++ < *index_p)
768                         continue;
769
770                 if (count >= *count_p)
771                         break;
772
773                 nd = ndl->ndl_node;
774                 if (copy_to_user(&dents_up[count].nde_id,
775                                      &nd->nd_id, sizeof(nd->nd_id)) ||
776                     copy_to_user(&dents_up[count].nde_state,
777                                      &nd->nd_state, sizeof(nd->nd_state)))
778                         return -EFAULT;
779
780                 count ++;
781         }
782
783         if (index <= *index_p)
784                 return -ENOENT;
785
786         *count_p = count;
787         *index_p = index;
788
789         return 0;
790 }
791
792 int
793 lstcon_group_info(char *name, lstcon_ndlist_ent_t __user *gents_p,
794                   int *index_p, int *count_p,
795                   lstcon_node_ent_t __user *dents_up)
796 {
797         lstcon_ndlist_ent_t *gentp;
798         lstcon_group_t      *grp;
799         lstcon_ndlink_t     *ndl;
800         int                  rc;
801
802         rc = lstcon_group_find(name, &grp);
803         if (rc != 0) {
804                 CDEBUG(D_NET, "Can't find group %s\n", name);
805                 return rc;
806         }
807
808         if (dents_up != NULL) {
809                 /* verbose query */
810                 rc = lstcon_nodes_getent(&grp->grp_ndl_list,
811                                          index_p, count_p, dents_up);
812                 lstcon_group_put(grp);
813
814                 return rc;
815         }
816
817         /* non-verbose query */
818         LIBCFS_ALLOC(gentp, sizeof(lstcon_ndlist_ent_t));
819         if (gentp == NULL) {
820                 CERROR("Can't allocate ndlist_ent\n");
821                 lstcon_group_put(grp);
822
823                 return -ENOMEM;
824         }
825
826         memset(gentp, 0, sizeof(lstcon_ndlist_ent_t));
827
828         list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link)
829                 LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp);
830
831         rc = copy_to_user(gents_p, gentp,
832                           sizeof(lstcon_ndlist_ent_t)) ? -EFAULT : 0;
833
834         LIBCFS_FREE(gentp, sizeof(lstcon_ndlist_ent_t));
835
836         lstcon_group_put(grp);
837
838         return 0;
839 }
840
841 static int
842 lstcon_batch_find(const char *name, lstcon_batch_t **batpp)
843 {
844         lstcon_batch_t *bat;
845
846         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
847                 if (strncmp(bat->bat_name, name, LST_NAME_SIZE) == 0) {
848                         *batpp = bat;
849                         return 0;
850                 }
851         }
852
853         return -ENOENT;
854 }
855
856 int
857 lstcon_batch_add(char *name)
858 {
859         lstcon_batch_t   *bat;
860         int               i;
861         int               rc;
862
863         rc = (lstcon_batch_find(name, &bat) == 0)? -EEXIST: 0;
864         if (rc != 0) {
865                 CDEBUG(D_NET, "Batch %s already exists\n", name);
866                 return rc;
867         }
868
869         LIBCFS_ALLOC(bat, sizeof(lstcon_batch_t));
870         if (bat == NULL) {
871                 CERROR("Can't allocate descriptor for batch %s\n", name);
872                 return -ENOMEM;
873         }
874
875         LIBCFS_ALLOC(bat->bat_cli_hash,
876                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
877         if (bat->bat_cli_hash == NULL) {
878                 CERROR("Can't allocate hash for batch %s\n", name);
879                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
880
881                 return -ENOMEM;
882         }
883
884         LIBCFS_ALLOC(bat->bat_srv_hash,
885                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
886         if (bat->bat_srv_hash == NULL) {
887                 CERROR("Can't allocate hash for batch %s\n", name);
888                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
889                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
890
891                 return -ENOMEM;
892         }
893
894         if (strlen(name) > sizeof(bat->bat_name)-1) {
895                 LIBCFS_FREE(bat->bat_srv_hash, LST_NODE_HASHSIZE);
896                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
897                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
898                 return -E2BIG;
899         }
900         strncpy(bat->bat_name, name, sizeof(bat->bat_name));
901         bat->bat_hdr.tsb_index = 0;
902         bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie;
903
904         bat->bat_ntest = 0;
905         bat->bat_state = LST_BATCH_IDLE;
906
907         INIT_LIST_HEAD(&bat->bat_cli_list);
908         INIT_LIST_HEAD(&bat->bat_srv_list);
909         INIT_LIST_HEAD(&bat->bat_test_list);
910         INIT_LIST_HEAD(&bat->bat_trans_list);
911
912         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
913                 INIT_LIST_HEAD(&bat->bat_cli_hash[i]);
914                 INIT_LIST_HEAD(&bat->bat_srv_hash[i]);
915         }
916
917         list_add_tail(&bat->bat_link, &console_session.ses_bat_list);
918
919         return rc;
920 }
921
922 int
923 lstcon_batch_list(int index, int len, char __user *name_up)
924 {
925         lstcon_batch_t *bat;
926
927         LASSERT(name_up != NULL);
928         LASSERT(index >= 0);
929
930         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
931                 if (index-- == 0) {
932                         return copy_to_user(name_up, bat->bat_name, len) ?
933                                             -EFAULT : 0;
934                 }
935         }
936
937         return -ENOENT;
938 }
939
940 int
941 lstcon_batch_info(char *name, lstcon_test_batch_ent_t __user *ent_up,
942                   int server, int testidx, int *index_p, int *ndent_p,
943                   lstcon_node_ent_t __user *dents_up)
944 {
945         lstcon_test_batch_ent_t *entp;
946         struct list_head        *clilst;
947         struct list_head        *srvlst;
948         lstcon_test_t           *test = NULL;
949         lstcon_batch_t          *bat;
950         lstcon_ndlink_t         *ndl;
951         int                      rc;
952
953         rc = lstcon_batch_find(name, &bat);
954         if (rc != 0) {
955                 CDEBUG(D_NET, "Can't find batch %s\n", name);
956                 return -ENOENT;
957         }
958
959         if (testidx > 0) {
960                 /* query test, test index start from 1 */
961                 list_for_each_entry(test, &bat->bat_test_list, tes_link) {
962                         if (testidx-- == 1)
963                                 break;
964                 }
965
966                 if (testidx > 0) {
967                         CDEBUG(D_NET, "Can't find specified test in batch\n");
968                         return -ENOENT;
969                 }
970         }
971
972         clilst = (test == NULL) ? &bat->bat_cli_list :
973                                   &test->tes_src_grp->grp_ndl_list;
974         srvlst = (test == NULL) ? &bat->bat_srv_list :
975                                   &test->tes_dst_grp->grp_ndl_list;
976
977         if (dents_up != NULL) {
978                 rc = lstcon_nodes_getent((server ? srvlst: clilst),
979                                          index_p, ndent_p, dents_up);
980                 return rc;
981         }
982
983         /* non-verbose query */
984         LIBCFS_ALLOC(entp, sizeof(lstcon_test_batch_ent_t));
985         if (entp == NULL)
986                 return -ENOMEM;
987
988         memset(entp, 0, sizeof(lstcon_test_batch_ent_t));
989
990         if (test == NULL) {
991                 entp->u.tbe_batch.bae_ntest = bat->bat_ntest;
992                 entp->u.tbe_batch.bae_state = bat->bat_state;
993
994         } else {
995
996                 entp->u.tbe_test.tse_type   = test->tes_type;
997                 entp->u.tbe_test.tse_loop   = test->tes_loop;
998                 entp->u.tbe_test.tse_concur = test->tes_concur;
999         }
1000
1001         list_for_each_entry(ndl, clilst, ndl_link)
1002                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle);
1003
1004         list_for_each_entry(ndl, srvlst, ndl_link)
1005                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle);
1006
1007         rc = copy_to_user(ent_up, entp,
1008                           sizeof(lstcon_test_batch_ent_t)) ? -EFAULT : 0;
1009
1010         LIBCFS_FREE(entp, sizeof(lstcon_test_batch_ent_t));
1011
1012         return rc;
1013 }
1014
1015 static int
1016 lstcon_batrpc_condition(int transop, lstcon_node_t *nd, void *arg)
1017 {
1018         switch (transop) {
1019         case LST_TRANS_TSBRUN:
1020                 if (nd->nd_state != LST_NODE_ACTIVE)
1021                         return -ENETDOWN;
1022                 break;
1023
1024         case LST_TRANS_TSBSTOP:
1025                 if (nd->nd_state != LST_NODE_ACTIVE)
1026                         return 0;
1027                 break;
1028
1029         case LST_TRANS_TSBCLIQRY:
1030         case LST_TRANS_TSBSRVQRY:
1031                 break;
1032         }
1033
1034         return 1;
1035 }
1036
1037 static int
1038 lstcon_batch_op(lstcon_batch_t *bat, int transop,
1039                 struct list_head __user *result_up)
1040 {
1041         lstcon_rpc_trans_t *trans;
1042         int                 rc;
1043
1044         rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list,
1045                                      &bat->bat_trans_list, transop,
1046                                      bat, lstcon_batrpc_condition, &trans);
1047         if (rc != 0) {
1048                 CERROR("Can't create transaction: %d\n", rc);
1049                 return rc;
1050         }
1051
1052         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1053
1054         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1055
1056         lstcon_rpc_trans_destroy(trans);
1057
1058         return rc;
1059 }
1060
1061 int
1062 lstcon_batch_run(char *name, int timeout, struct list_head __user *result_up)
1063 {
1064         lstcon_batch_t *bat;
1065         int             rc;
1066
1067         if (lstcon_batch_find(name, &bat) != 0) {
1068                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1069                 return -ENOENT;
1070         }
1071
1072         bat->bat_arg = timeout;
1073
1074         rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up);
1075
1076         /* mark batch as running if it's started in any node */
1077         if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0) != 0)
1078                 bat->bat_state = LST_BATCH_RUNNING;
1079
1080         return rc;
1081 }
1082
1083 int
1084 lstcon_batch_stop(char *name, int force, struct list_head __user *result_up)
1085 {
1086         lstcon_batch_t *bat;
1087         int             rc;
1088
1089         if (lstcon_batch_find(name, &bat) != 0) {
1090                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1091                 return -ENOENT;
1092         }
1093
1094         bat->bat_arg = force;
1095
1096         rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up);
1097
1098         /* mark batch as stopped if all RPCs finished */
1099         if (lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0) == 0)
1100                 bat->bat_state = LST_BATCH_IDLE;
1101
1102         return rc;
1103 }
1104
1105 static void
1106 lstcon_batch_destroy(lstcon_batch_t *bat)
1107 {
1108         lstcon_ndlink_t    *ndl;
1109         lstcon_test_t      *test;
1110         int                 i;
1111
1112         list_del(&bat->bat_link);
1113
1114         while (!list_empty(&bat->bat_test_list)) {
1115                 test = list_entry(bat->bat_test_list.next,
1116                                   lstcon_test_t, tes_link);
1117                 LASSERT(list_empty(&test->tes_trans_list));
1118
1119                 list_del(&test->tes_link);
1120
1121                 lstcon_group_put(test->tes_src_grp);
1122                 lstcon_group_put(test->tes_dst_grp);
1123
1124                 LIBCFS_FREE(test, offsetof(lstcon_test_t,
1125                                            tes_param[test->tes_paramlen]));
1126         }
1127
1128         LASSERT(list_empty(&bat->bat_trans_list));
1129
1130         while (!list_empty(&bat->bat_cli_list)) {
1131                 ndl = list_entry(bat->bat_cli_list.next,
1132                                  lstcon_ndlink_t, ndl_link);
1133                 list_del_init(&ndl->ndl_link);
1134
1135                 lstcon_ndlink_release(ndl);
1136         }
1137
1138         while (!list_empty(&bat->bat_srv_list)) {
1139                 ndl = list_entry(bat->bat_srv_list.next,
1140                                  lstcon_ndlink_t, ndl_link);
1141                 list_del_init(&ndl->ndl_link);
1142
1143                 lstcon_ndlink_release(ndl);
1144         }
1145
1146         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1147                 LASSERT(list_empty(&bat->bat_cli_hash[i]));
1148                 LASSERT(list_empty(&bat->bat_srv_hash[i]));
1149         }
1150
1151         LIBCFS_FREE(bat->bat_cli_hash,
1152                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1153         LIBCFS_FREE(bat->bat_srv_hash,
1154                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1155         LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
1156 }
1157
1158 static int
1159 lstcon_testrpc_condition(int transop, lstcon_node_t *nd, void *arg)
1160 {
1161         lstcon_test_t    *test;
1162         lstcon_batch_t   *batch;
1163         lstcon_ndlink_t  *ndl;
1164         struct list_head *hash;
1165         struct list_head *head;
1166
1167         test = (lstcon_test_t *)arg;
1168         LASSERT(test != NULL);
1169
1170         batch = test->tes_batch;
1171         LASSERT(batch != NULL);
1172
1173         if (test->tes_oneside &&
1174             transop == LST_TRANS_TSBSRVADD)
1175                 return 0;
1176
1177         if (nd->nd_state != LST_NODE_ACTIVE)
1178                 return -ENETDOWN;
1179
1180         if (transop == LST_TRANS_TSBCLIADD) {
1181                 hash = batch->bat_cli_hash;
1182                 head = &batch->bat_cli_list;
1183
1184         } else {
1185                 LASSERT (transop == LST_TRANS_TSBSRVADD);
1186
1187                 hash = batch->bat_srv_hash;
1188                 head = &batch->bat_srv_list;
1189         }
1190
1191         LASSERT (nd->nd_id.nid != LNET_NID_ANY);
1192
1193         if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1) != 0)
1194                 return -ENOMEM;
1195
1196         if (list_empty(&ndl->ndl_link))
1197                 list_add_tail(&ndl->ndl_link, head);
1198
1199         return 1;
1200 }
1201
1202 static int
1203 lstcon_test_nodes_add(lstcon_test_t *test, struct list_head __user *result_up)
1204 {
1205         lstcon_rpc_trans_t     *trans;
1206         lstcon_group_t         *grp;
1207         int                     transop;
1208         int                     rc;
1209
1210         LASSERT (test->tes_src_grp != NULL);
1211         LASSERT (test->tes_dst_grp != NULL);
1212
1213         transop = LST_TRANS_TSBSRVADD;
1214         grp  = test->tes_dst_grp;
1215 again:
1216         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
1217                                      &test->tes_trans_list, transop,
1218                                      test, lstcon_testrpc_condition, &trans);
1219         if (rc != 0) {
1220                 CERROR("Can't create transaction: %d\n", rc);
1221                 return rc;
1222         }
1223
1224         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1225
1226         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1227             lstcon_trans_stat()->trs_fwk_errno != 0) {
1228                 lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1229
1230                 lstcon_rpc_trans_destroy(trans);
1231                 /* return if any error */
1232                 CDEBUG(D_NET, "Failed to add test %s, "
1233                               "RPC error %d, framework error %d\n",
1234                        transop == LST_TRANS_TSBCLIADD ? "client" : "server",
1235                        lstcon_trans_stat()->trs_rpc_errno,
1236                        lstcon_trans_stat()->trs_fwk_errno);
1237
1238                 return rc;
1239         }
1240
1241         lstcon_rpc_trans_destroy(trans);
1242
1243         if (transop == LST_TRANS_TSBCLIADD)
1244                 return rc;
1245
1246         transop = LST_TRANS_TSBCLIADD;
1247         grp = test->tes_src_grp;
1248         test->tes_cliidx = 0;
1249
1250         /* requests to test clients */
1251         goto again;
1252 }
1253
1254 static int
1255 lstcon_verify_batch(const char *name, lstcon_batch_t **batch)
1256 {
1257         int rc;
1258
1259         rc = lstcon_batch_find(name, batch);
1260         if (rc != 0) {
1261                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1262                 return rc;
1263         }
1264
1265         if ((*batch)->bat_state != LST_BATCH_IDLE) {
1266                 CDEBUG(D_NET, "Can't change running batch %s\n", name);
1267                 return -EINVAL;
1268         }
1269
1270         return 0;
1271 }
1272
1273 static int
1274 lstcon_verify_group(const char *name, lstcon_group_t **grp)
1275 {
1276         int                     rc;
1277         lstcon_ndlink_t         *ndl;
1278
1279         rc = lstcon_group_find(name, grp);
1280         if (rc != 0) {
1281                 CDEBUG(D_NET, "can't find group %s\n", name);
1282                 return rc;
1283         }
1284
1285         list_for_each_entry(ndl, &(*grp)->grp_ndl_list, ndl_link) {
1286                 if (ndl->ndl_node->nd_state == LST_NODE_ACTIVE) {
1287                         return 0;
1288                 }
1289         }
1290
1291         CDEBUG(D_NET, "Group %s has no ACTIVE nodes\n", name);
1292
1293         return -EINVAL;
1294 }
1295
1296 int
1297 lstcon_test_add(char *batch_name, int type, int loop,
1298                 int concur, int dist, int span,
1299                 char *src_name, char *dst_name,
1300                 void *param, int paramlen, int *retp,
1301                 struct list_head __user *result_up)
1302 {
1303         lstcon_test_t    *test   = NULL;
1304         int              rc;
1305         lstcon_group_t   *src_grp = NULL;
1306         lstcon_group_t   *dst_grp = NULL;
1307         lstcon_batch_t   *batch = NULL;
1308
1309         /*
1310          * verify that a batch of the given name exists, and the groups
1311          * that will be part of the batch exist and have at least one
1312          * active node
1313          */
1314         rc = lstcon_verify_batch(batch_name, &batch);
1315         if (rc != 0)
1316                 goto out;
1317
1318         rc = lstcon_verify_group(src_name, &src_grp);
1319         if (rc != 0)
1320                 goto out;
1321
1322         rc = lstcon_verify_group(dst_name, &dst_grp);
1323         if (rc != 0)
1324                 goto out;
1325
1326         if (dst_grp->grp_userland)
1327                 *retp = 1;
1328
1329         LIBCFS_ALLOC(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1330         if (!test) {
1331                 CERROR("Can't allocate test descriptor\n");
1332                 rc = -ENOMEM;
1333
1334                 goto out;
1335         }
1336
1337         memset(test, 0, offsetof(lstcon_test_t, tes_param[paramlen]));
1338         test->tes_hdr.tsb_id    = batch->bat_hdr.tsb_id;
1339         test->tes_batch         = batch;
1340         test->tes_type          = type;
1341         test->tes_oneside       = 0; /* TODO */
1342         test->tes_loop          = loop;
1343         test->tes_concur        = concur;
1344         test->tes_stop_onerr    = 1; /* TODO */
1345         test->tes_span          = span;
1346         test->tes_dist          = dist;
1347         test->tes_cliidx        = 0; /* just used for creating RPC */
1348         test->tes_src_grp       = src_grp;
1349         test->tes_dst_grp       = dst_grp;
1350         INIT_LIST_HEAD(&test->tes_trans_list);
1351
1352         if (param != NULL) {
1353                 test->tes_paramlen = paramlen;
1354                 memcpy(&test->tes_param[0], param, paramlen);
1355         }
1356
1357         rc = lstcon_test_nodes_add(test, result_up);
1358
1359         if (rc != 0)
1360                 goto out;
1361
1362         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1363             lstcon_trans_stat()->trs_fwk_errno != 0)
1364                 CDEBUG(D_NET, "Failed to add test %d to batch %s\n", type,
1365                        batch_name);
1366
1367         /* add to test list anyway, so user can check what's going on */
1368         list_add_tail(&test->tes_link, &batch->bat_test_list);
1369
1370         batch->bat_ntest++;
1371         test->tes_hdr.tsb_index = batch->bat_ntest;
1372
1373         /*  hold groups so nobody can change them */
1374         return rc;
1375 out:
1376         if (test != NULL)
1377                 LIBCFS_FREE(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1378
1379         if (dst_grp != NULL)
1380                 lstcon_group_put(dst_grp);
1381
1382         if (src_grp != NULL)
1383                 lstcon_group_put(src_grp);
1384
1385         return rc;
1386 }
1387
1388 static int
1389 lstcon_test_find(lstcon_batch_t *batch, int idx, lstcon_test_t **testpp)
1390 {
1391         lstcon_test_t *test;
1392
1393         list_for_each_entry(test, &batch->bat_test_list, tes_link) {
1394                 if (idx == test->tes_hdr.tsb_index) {
1395                         *testpp = test;
1396                         return 0;
1397                 }
1398         }
1399
1400         return -ENOENT;
1401 }
1402
1403 static int
1404 lstcon_tsbrpc_readent(int transop, srpc_msg_t *msg,
1405                       lstcon_rpc_ent_t __user *ent_up)
1406 {
1407         srpc_batch_reply_t *rep = &msg->msg_body.bat_reply;
1408
1409         LASSERT (transop == LST_TRANS_TSBCLIQRY ||
1410                  transop == LST_TRANS_TSBSRVQRY);
1411
1412         /* positive errno, framework error code */
1413         if (copy_to_user(&ent_up->rpe_priv[0],
1414                              &rep->bar_active, sizeof(rep->bar_active)))
1415                 return -EFAULT;
1416
1417         return 0;
1418 }
1419
1420 int
1421 lstcon_test_batch_query(char *name, int testidx, int client,
1422                         int timeout, struct list_head __user *result_up)
1423 {
1424         lstcon_rpc_trans_t *trans;
1425         struct list_head   *translist;
1426         struct list_head   *ndlist;
1427         lstcon_tsb_hdr_t   *hdr;
1428         lstcon_batch_t     *batch;
1429         lstcon_test_t      *test = NULL;
1430         int                 transop;
1431         int                 rc;
1432
1433         rc = lstcon_batch_find(name, &batch);
1434         if (rc != 0) {
1435                 CDEBUG(D_NET, "Can't find batch: %s\n", name);
1436                 return rc;
1437         }
1438
1439         if (testidx == 0) {
1440                 translist = &batch->bat_trans_list;
1441                 ndlist    = &batch->bat_cli_list;
1442                 hdr       = &batch->bat_hdr;
1443
1444         } else {
1445                 /* query specified test only */
1446                 rc = lstcon_test_find(batch, testidx, &test);
1447                 if (rc != 0) {
1448                         CDEBUG(D_NET, "Can't find test: %d\n", testidx);
1449                         return rc;
1450                 }
1451
1452                 translist = &test->tes_trans_list;
1453                 ndlist    = &test->tes_src_grp->grp_ndl_list;
1454                 hdr       = &test->tes_hdr;
1455         }
1456
1457         transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY;
1458
1459         rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr,
1460                                      lstcon_batrpc_condition, &trans);
1461         if (rc != 0) {
1462                 CERROR("Can't create transaction: %d\n", rc);
1463                 return rc;
1464         }
1465
1466         lstcon_rpc_trans_postwait(trans, timeout);
1467
1468         if (testidx == 0 && /* query a batch, not a test */
1469             lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) == 0 &&
1470             lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0) == 0) {
1471                 /* all RPCs finished, and no active test */
1472                 batch->bat_state = LST_BATCH_IDLE;
1473         }
1474
1475         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1476                                           lstcon_tsbrpc_readent);
1477         lstcon_rpc_trans_destroy(trans);
1478
1479         return rc;
1480 }
1481
1482 static int
1483 lstcon_statrpc_readent(int transop, srpc_msg_t *msg,
1484                        lstcon_rpc_ent_t __user *ent_up)
1485 {
1486         srpc_stat_reply_t *rep = &msg->msg_body.stat_reply;
1487         sfw_counters_t __user  *sfwk_stat;
1488         srpc_counters_t __user *srpc_stat;
1489         lnet_counters_t __user *lnet_stat;
1490
1491         if (rep->str_status != 0)
1492                 return 0;
1493
1494         sfwk_stat = (sfw_counters_t __user *)&ent_up->rpe_payload[0];
1495         srpc_stat = (srpc_counters_t __user *)
1496                 ((char __user *)sfwk_stat + sizeof(*sfwk_stat));
1497         lnet_stat = (lnet_counters_t __user *)
1498                 ((char __user *)srpc_stat + sizeof(*srpc_stat));
1499
1500         if (copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) ||
1501             copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) ||
1502             copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat)))
1503                 return -EFAULT;
1504
1505         return 0;
1506 }
1507
1508 static int
1509 lstcon_ndlist_stat(struct list_head *ndlist,
1510                    int timeout, struct list_head __user *result_up)
1511 {
1512         struct list_head    head;
1513         lstcon_rpc_trans_t *trans;
1514         int                 rc;
1515
1516         INIT_LIST_HEAD(&head);
1517
1518         rc = lstcon_rpc_trans_ndlist(ndlist, &head,
1519                                      LST_TRANS_STATQRY, NULL, NULL, &trans);
1520         if (rc != 0) {
1521                 CERROR("Can't create transaction: %d\n", rc);
1522                 return rc;
1523         }
1524
1525         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1526
1527         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1528                                           lstcon_statrpc_readent);
1529         lstcon_rpc_trans_destroy(trans);
1530
1531         return rc;
1532 }
1533
1534 int
1535 lstcon_group_stat(char *grp_name, int timeout,
1536                   struct list_head __user *result_up)
1537 {
1538         lstcon_group_t     *grp;
1539         int                 rc;
1540
1541         rc = lstcon_group_find(grp_name, &grp);
1542         if (rc != 0) {
1543                 CDEBUG(D_NET, "Can't find group %s\n", grp_name);
1544                 return rc;
1545         }
1546
1547         rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up);
1548
1549         lstcon_group_put(grp);
1550
1551         return rc;
1552 }
1553
1554 int
1555 lstcon_nodes_stat(int count, lnet_process_id_t __user *ids_up,
1556                   int timeout, struct list_head __user *result_up)
1557 {
1558         lstcon_ndlink_t         *ndl;
1559         lstcon_group_t          *tmp;
1560         lnet_process_id_t        id;
1561         int                      i;
1562         int                      rc;
1563
1564         rc = lstcon_group_alloc(NULL, &tmp);
1565         if (rc != 0) {
1566                 CERROR("Out of memory\n");
1567                 return -ENOMEM;
1568         }
1569
1570         for (i = 0 ; i < count; i++) {
1571                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1572                         rc = -EFAULT;
1573                         break;
1574                 }
1575
1576                 /* add to tmp group */
1577                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2);
1578                 if (rc != 0) {
1579                         CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET,
1580                                "Failed to find or create %s: %d\n",
1581                                libcfs_id2str(id), rc);
1582                         break;
1583                 }
1584         }
1585
1586         if (rc != 0) {
1587                 lstcon_group_put(tmp);
1588                 return rc;
1589         }
1590
1591         rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up);
1592
1593         lstcon_group_put(tmp);
1594
1595         return rc;
1596 }
1597
1598 static int
1599 lstcon_debug_ndlist(struct list_head *ndlist,
1600                     struct list_head *translist,
1601                     int timeout, struct list_head __user *result_up)
1602 {
1603         lstcon_rpc_trans_t *trans;
1604         int                 rc;
1605
1606         rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY,
1607                                      NULL, lstcon_sesrpc_condition, &trans);
1608         if (rc != 0) {
1609                 CERROR("Can't create transaction: %d\n", rc);
1610                 return rc;
1611         }
1612
1613         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1614
1615         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1616                                           lstcon_sesrpc_readent);
1617         lstcon_rpc_trans_destroy(trans);
1618
1619         return rc;
1620 }
1621
1622 int
1623 lstcon_session_debug(int timeout, struct list_head __user *result_up)
1624 {
1625         return lstcon_debug_ndlist(&console_session.ses_ndl_list,
1626                                    NULL, timeout, result_up);
1627 }
1628
1629 int
1630 lstcon_batch_debug(int timeout, char *name,
1631                    int client, struct list_head __user *result_up)
1632 {
1633         lstcon_batch_t *bat;
1634         int             rc;
1635
1636         rc = lstcon_batch_find(name, &bat);
1637         if (rc != 0)
1638                 return -ENOENT;
1639
1640         rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list :
1641                                           &bat->bat_srv_list,
1642                                  NULL, timeout, result_up);
1643
1644         return rc;
1645 }
1646
1647 int
1648 lstcon_group_debug(int timeout, char *name,
1649                    struct list_head __user *result_up)
1650 {
1651         lstcon_group_t *grp;
1652         int             rc;
1653
1654         rc = lstcon_group_find(name, &grp);
1655         if (rc != 0)
1656                 return -ENOENT;
1657
1658         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1659                                  timeout, result_up);
1660         lstcon_group_put(grp);
1661
1662         return rc;
1663 }
1664
1665 int
1666 lstcon_nodes_debug(int timeout,
1667                    int count, lnet_process_id_t __user *ids_up,
1668                    struct list_head __user *result_up)
1669 {
1670         lnet_process_id_t  id;
1671         lstcon_ndlink_t   *ndl;
1672         lstcon_group_t    *grp;
1673         int                i;
1674         int                rc;
1675
1676         rc = lstcon_group_alloc(NULL, &grp);
1677         if (rc != 0) {
1678                 CDEBUG(D_NET, "Out of memory\n");
1679                 return rc;
1680         }
1681
1682         for (i = 0; i < count; i++) {
1683                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1684                         rc = -EFAULT;
1685                         break;
1686                 }
1687
1688                 /* node is added to tmp group */
1689                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 1);
1690                 if (rc != 0) {
1691                         CERROR("Can't create node link\n");
1692                         break;
1693                 }
1694         }
1695
1696         if (rc != 0) {
1697                 lstcon_group_put(grp);
1698                 return rc;
1699         }
1700
1701         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1702                                  timeout, result_up);
1703
1704         lstcon_group_put(grp);
1705
1706         return rc;
1707 }
1708
1709 int
1710 lstcon_session_match(lst_sid_t sid)
1711 {
1712         return (console_session.ses_id.ses_nid   == sid.ses_nid &&
1713                 console_session.ses_id.ses_stamp == sid.ses_stamp) ?  1: 0;
1714 }
1715
1716 static void
1717 lstcon_new_session_id(lst_sid_t *sid)
1718 {
1719         lnet_process_id_t      id;
1720
1721         LASSERT (console_session.ses_state == LST_SESSION_NONE);
1722
1723         LNetGetId(1, &id);
1724         sid->ses_nid   = id.nid;
1725         sid->ses_stamp = cfs_time_current();
1726 }
1727
1728 int
1729 lstcon_session_new(char *name, int key, unsigned feats,
1730                    int timeout, int force, lst_sid_t __user *sid_up)
1731 {
1732         int     rc = 0;
1733         int     i;
1734
1735         if (console_session.ses_state != LST_SESSION_NONE) {
1736                 /* session exists */
1737                 if (!force) {
1738                         CNETERR("Session %s already exists\n",
1739                                 console_session.ses_name);
1740                         return -EEXIST;
1741                 }
1742
1743                 rc = lstcon_session_end();
1744
1745                 /* lstcon_session_end() only return local error */
1746                 if  (rc != 0)
1747                         return rc;
1748         }
1749
1750         if ((feats & ~LST_FEATS_MASK) != 0) {
1751                 CNETERR("Unknown session features %x\n",
1752                         (feats & ~LST_FEATS_MASK));
1753                 return -EINVAL;
1754         }
1755
1756         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
1757                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
1758
1759         lstcon_new_session_id(&console_session.ses_id);
1760
1761         console_session.ses_key     = key;
1762         console_session.ses_state   = LST_SESSION_ACTIVE;
1763         console_session.ses_force   = !!force;
1764         console_session.ses_features = feats;
1765         console_session.ses_feats_updated = 0;
1766         console_session.ses_timeout = (timeout <= 0) ?
1767                                       LST_CONSOLE_TIMEOUT : timeout;
1768
1769         if (strlen(name) > sizeof(console_session.ses_name)-1)
1770                 return -E2BIG;
1771         strlcpy(console_session.ses_name, name,
1772                 sizeof(console_session.ses_name));
1773
1774         rc = lstcon_batch_add(LST_DEFAULT_BATCH);
1775         if (rc != 0)
1776                 return rc;
1777
1778         rc = lstcon_rpc_pinger_start();
1779         if (rc != 0) {
1780                 lstcon_batch_t *bat = NULL;
1781
1782                 lstcon_batch_find(LST_DEFAULT_BATCH, &bat);
1783                 lstcon_batch_destroy(bat);
1784
1785                 return rc;
1786         }
1787
1788         if (copy_to_user(sid_up, &console_session.ses_id,
1789                              sizeof(lst_sid_t)) == 0)
1790                 return rc;
1791
1792         lstcon_session_end();
1793
1794         return -EFAULT;
1795 }
1796
1797 int
1798 lstcon_session_info(lst_sid_t __user *sid_up, int __user *key_up,
1799                     unsigned __user *featp,
1800                     lstcon_ndlist_ent_t __user *ndinfo_up,
1801                     char __user *name_up, int len)
1802 {
1803         lstcon_ndlist_ent_t *entp;
1804         lstcon_ndlink_t     *ndl;
1805         int                  rc = 0;
1806
1807         if (console_session.ses_state != LST_SESSION_ACTIVE)
1808                 return -ESRCH;
1809
1810         LIBCFS_ALLOC(entp, sizeof(*entp));
1811         if (entp == NULL)
1812                 return -ENOMEM;
1813
1814         memset(entp, 0, sizeof(*entp));
1815
1816         list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link)
1817                 LST_NODE_STATE_COUNTER(ndl->ndl_node, entp);
1818
1819         if (copy_to_user(sid_up, &console_session.ses_id,
1820                              sizeof(lst_sid_t)) ||
1821             copy_to_user(key_up, &console_session.ses_key,
1822                              sizeof(*key_up)) ||
1823             copy_to_user(featp, &console_session.ses_features,
1824                              sizeof(*featp)) ||
1825             copy_to_user(ndinfo_up, entp, sizeof(*entp)) ||
1826             copy_to_user(name_up, console_session.ses_name, len))
1827                 rc = -EFAULT;
1828
1829         LIBCFS_FREE(entp, sizeof(*entp));
1830
1831         return rc;
1832 }
1833
1834 int
1835 lstcon_session_end()
1836 {
1837         lstcon_rpc_trans_t *trans;
1838         lstcon_group_t     *grp;
1839         lstcon_batch_t     *bat;
1840         int                 rc = 0;
1841
1842         LASSERT (console_session.ses_state == LST_SESSION_ACTIVE);
1843
1844         rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list,
1845                                      NULL, LST_TRANS_SESEND, NULL,
1846                                      lstcon_sesrpc_condition, &trans);
1847         if (rc != 0) {
1848                 CERROR("Can't create transaction: %d\n", rc);
1849                 return rc;
1850         }
1851
1852         console_session.ses_shutdown = 1;
1853
1854         lstcon_rpc_pinger_stop();
1855
1856         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1857
1858         lstcon_rpc_trans_destroy(trans);
1859         /* User can do nothing even rpc failed, so go on */
1860
1861         /* waiting for orphan rpcs to die */
1862         lstcon_rpc_cleanup_wait();
1863
1864         console_session.ses_id    = LST_INVALID_SID;
1865         console_session.ses_state = LST_SESSION_NONE;
1866         console_session.ses_key   = 0;
1867         console_session.ses_force = 0;
1868         console_session.ses_feats_updated = 0;
1869
1870         /* destroy all batches */
1871         while (!list_empty(&console_session.ses_bat_list)) {
1872                 bat = list_entry(console_session.ses_bat_list.next,
1873                                  lstcon_batch_t, bat_link);
1874
1875                 lstcon_batch_destroy(bat);
1876         }
1877
1878         /* destroy all groups */
1879         while (!list_empty(&console_session.ses_grp_list)) {
1880                 grp = list_entry(console_session.ses_grp_list.next,
1881                                  lstcon_group_t, grp_link);
1882                 LASSERT(grp->grp_ref == 1);
1883
1884                 lstcon_group_put(grp);
1885         }
1886
1887         /* all nodes should be released */
1888         LASSERT(list_empty(&console_session.ses_ndl_list));
1889
1890         console_session.ses_shutdown = 0;
1891         console_session.ses_expired  = 0;
1892
1893         return rc;
1894 }
1895
1896 int
1897 lstcon_session_feats_check(unsigned feats)
1898 {
1899         int rc = 0;
1900
1901         if ((feats & ~LST_FEATS_MASK) != 0) {
1902                 CERROR("Can't support these features: %x\n",
1903                        (feats & ~LST_FEATS_MASK));
1904                 return -EPROTO;
1905         }
1906
1907         spin_lock(&console_session.ses_rpc_lock);
1908
1909         if (!console_session.ses_feats_updated) {
1910                 console_session.ses_feats_updated = 1;
1911                 console_session.ses_features = feats;
1912         }
1913
1914         if (console_session.ses_features != feats)
1915                 rc = -EPROTO;
1916
1917         spin_unlock(&console_session.ses_rpc_lock);
1918
1919         if (rc != 0) {
1920                 CERROR("remote features %x do not match with "
1921                        "session features %x of console\n",
1922                        feats, console_session.ses_features);
1923         }
1924
1925         return rc;
1926 }
1927
1928 static int
1929 lstcon_acceptor_handle (srpc_server_rpc_t *rpc)
1930 {
1931         srpc_msg_t        *rep  = &rpc->srpc_replymsg;
1932         srpc_msg_t        *req  = &rpc->srpc_reqstbuf->buf_msg;
1933         srpc_join_reqst_t *jreq = &req->msg_body.join_reqst;
1934         srpc_join_reply_t *jrep = &rep->msg_body.join_reply;
1935         lstcon_group_t    *grp  = NULL;
1936         lstcon_ndlink_t   *ndl;
1937         int                rc   = 0;
1938
1939         sfw_unpack_message(req);
1940
1941         mutex_lock(&console_session.ses_mutex);
1942
1943         jrep->join_sid = console_session.ses_id;
1944
1945         if (console_session.ses_id.ses_nid == LNET_NID_ANY) {
1946                 jrep->join_status = ESRCH;
1947                 goto out;
1948         }
1949
1950         if (lstcon_session_feats_check(req->msg_ses_feats) != 0) {
1951                 jrep->join_status = EPROTO;
1952                 goto out;
1953         }
1954
1955         if (jreq->join_sid.ses_nid != LNET_NID_ANY &&
1956              !lstcon_session_match(jreq->join_sid)) {
1957                 jrep->join_status = EBUSY;
1958                 goto out;
1959         }
1960
1961         if (lstcon_group_find(jreq->join_group, &grp) != 0) {
1962                 rc = lstcon_group_alloc(jreq->join_group, &grp);
1963                 if (rc != 0) {
1964                         CERROR("Out of memory\n");
1965                         goto out;
1966                 }
1967
1968                 list_add_tail(&grp->grp_link,
1969                               &console_session.ses_grp_list);
1970                 lstcon_group_addref(grp);
1971         }
1972
1973         if (grp->grp_ref > 2) {
1974                 /* Group in using */
1975                 jrep->join_status = EBUSY;
1976                 goto out;
1977         }
1978
1979         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0);
1980         if (rc == 0) {
1981                 jrep->join_status = EEXIST;
1982                 goto out;
1983         }
1984
1985         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1);
1986         if (rc != 0) {
1987                 CERROR("Out of memory\n");
1988                 goto out;
1989         }
1990
1991         ndl->ndl_node->nd_state   = LST_NODE_ACTIVE;
1992         ndl->ndl_node->nd_timeout = console_session.ses_timeout;
1993
1994         if (grp->grp_userland == 0)
1995                 grp->grp_userland = 1;
1996
1997         strlcpy(jrep->join_session, console_session.ses_name,
1998                 sizeof(jrep->join_session));
1999         jrep->join_timeout = console_session.ses_timeout;
2000         jrep->join_status  = 0;
2001
2002 out:
2003         rep->msg_ses_feats = console_session.ses_features;
2004         if (grp != NULL)
2005                 lstcon_group_put(grp);
2006
2007         mutex_unlock(&console_session.ses_mutex);
2008
2009         return rc;
2010 }
2011
2012 static srpc_service_t lstcon_acceptor_service;
2013 static void lstcon_init_acceptor_service(void)
2014 {
2015         /* initialize selftest console acceptor service table */
2016         lstcon_acceptor_service.sv_name    = "join session";
2017         lstcon_acceptor_service.sv_handler = lstcon_acceptor_handle;
2018         lstcon_acceptor_service.sv_id      = SRPC_SERVICE_JOIN;
2019         lstcon_acceptor_service.sv_wi_total = SFW_FRWK_WI_MAX;
2020 }
2021
2022 int lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_hdr *hdr);
2023
2024 DECLARE_IOCTL_HANDLER(lstcon_ioctl_handler, lstcon_ioctl_entry);
2025
2026 /* initialize console */
2027 int
2028 lstcon_console_init(void)
2029 {
2030         int     i;
2031         int     rc;
2032
2033         memset(&console_session, 0, sizeof(lstcon_session_t));
2034
2035         console_session.ses_id              = LST_INVALID_SID;
2036         console_session.ses_state           = LST_SESSION_NONE;
2037         console_session.ses_timeout         = 0;
2038         console_session.ses_force           = 0;
2039         console_session.ses_expired         = 0;
2040         console_session.ses_feats_updated   = 0;
2041         console_session.ses_features        = LST_FEATS_MASK;
2042         console_session.ses_laststamp       = cfs_time_current_sec();
2043
2044         mutex_init(&console_session.ses_mutex);
2045
2046         INIT_LIST_HEAD(&console_session.ses_ndl_list);
2047         INIT_LIST_HEAD(&console_session.ses_grp_list);
2048         INIT_LIST_HEAD(&console_session.ses_bat_list);
2049         INIT_LIST_HEAD(&console_session.ses_trans_list);
2050
2051         LIBCFS_ALLOC(console_session.ses_ndl_hash,
2052                      sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2053         if (console_session.ses_ndl_hash == NULL)
2054                 return -ENOMEM;
2055
2056         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
2057                 INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]);
2058
2059
2060         /* initialize acceptor service table */
2061         lstcon_init_acceptor_service();
2062
2063         rc = srpc_add_service(&lstcon_acceptor_service);
2064         LASSERT(rc != -EBUSY);
2065         if (rc != 0) {
2066                 LIBCFS_FREE(console_session.ses_ndl_hash,
2067                             sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2068                 return rc;
2069         }
2070
2071         rc = srpc_service_add_buffers(&lstcon_acceptor_service,
2072                                       lstcon_acceptor_service.sv_wi_total);
2073         if (rc != 0) {
2074                 rc = -ENOMEM;
2075                 goto out;
2076         }
2077
2078         rc = libcfs_register_ioctl(&lstcon_ioctl_handler);
2079
2080         if (rc == 0) {
2081                 lstcon_rpc_module_init();
2082                 return 0;
2083         }
2084
2085 out:
2086         srpc_shutdown_service(&lstcon_acceptor_service);
2087         srpc_remove_service(&lstcon_acceptor_service);
2088
2089         LIBCFS_FREE(console_session.ses_ndl_hash,
2090                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2091
2092         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2093
2094         return rc;
2095 }
2096
2097 int
2098 lstcon_console_fini(void)
2099 {
2100         int     i;
2101
2102         libcfs_deregister_ioctl(&lstcon_ioctl_handler);
2103
2104         mutex_lock(&console_session.ses_mutex);
2105
2106         srpc_shutdown_service(&lstcon_acceptor_service);
2107         srpc_remove_service(&lstcon_acceptor_service);
2108
2109         if (console_session.ses_state != LST_SESSION_NONE)
2110                 lstcon_session_end();
2111
2112         lstcon_rpc_module_fini();
2113
2114         mutex_unlock(&console_session.ses_mutex);
2115
2116         LASSERT(list_empty(&console_session.ses_ndl_list));
2117         LASSERT(list_empty(&console_session.ses_grp_list));
2118         LASSERT(list_empty(&console_session.ses_bat_list));
2119         LASSERT(list_empty(&console_session.ses_trans_list));
2120
2121         for (i = 0; i < LST_NODE_HASHSIZE; i++)
2122                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
2123
2124         LIBCFS_FREE(console_session.ses_ndl_hash,
2125                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2126
2127         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2128
2129         return 0;
2130 }
2131