Whamcloud - gitweb
0be173d07baba1dc3612feb030e8e2301b99bcec
[fs/lustre-release.git] / lnet / selftest / console.c
1 /*
2  * GPL HEADER START
3  *
4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 only,
8  * as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13  * General Public License version 2 for more details (a copy is included
14  * in the LICENSE file that accompanied this code).
15  *
16  * You should have received a copy of the GNU General Public License
17  * version 2 along with this program; If not, see
18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
19  *
20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21  * CA 95054 USA or visit www.sun.com if you need additional information or
22  * have any questions.
23  *
24  * GPL HEADER END
25  */
26 /*
27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
28  * Use is subject to license terms.
29  *
30  * Copyright (c) 2012, 2014, Intel Corporation.
31  */
32 /*
33  * This file is part of Lustre, http://www.lustre.org/
34  * Lustre is a trademark of Sun Microsystems, Inc.
35  *
36  * lnet/selftest/conctl.c
37  *
38  * Infrastructure of LST console
39  *
40  * Author: Liang Zhen <liangzhen@clusterfs.com>
41  */
42
43 #ifdef __KERNEL__
44
45 #include <libcfs/libcfs.h>
46 #include <lnet/lib-lnet.h>
47 #include "console.h"
48 #include "conrpc.h"
49
50 #define LST_NODE_STATE_COUNTER(nd, p)                   \
51 do {                                                    \
52         if ((nd)->nd_state == LST_NODE_ACTIVE)          \
53                 (p)->nle_nactive ++;                    \
54         else if ((nd)->nd_state == LST_NODE_BUSY)       \
55                 (p)->nle_nbusy ++;                      \
56         else if ((nd)->nd_state == LST_NODE_DOWN)       \
57                 (p)->nle_ndown ++;                      \
58         else                                            \
59                 (p)->nle_nunknown ++;                   \
60         (p)->nle_nnode ++;                              \
61 } while (0)
62
63 lstcon_session_t        console_session;
64
65 void
66 lstcon_node_get(lstcon_node_t *nd)
67 {
68         LASSERT (nd->nd_ref >= 1);
69
70         nd->nd_ref++;
71 }
72
73 static int
74 lstcon_node_find(lnet_process_id_t id, lstcon_node_t **ndpp, int create)
75 {
76         lstcon_ndlink_t *ndl;
77         unsigned int     idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE;
78
79         LASSERT(id.nid != LNET_NID_ANY);
80
81         list_for_each_entry(ndl, &console_session.ses_ndl_hash[idx],
82                             ndl_hlink) {
83                 if (ndl->ndl_node->nd_id.nid != id.nid ||
84                     ndl->ndl_node->nd_id.pid != id.pid)
85                         continue;
86
87                 lstcon_node_get(ndl->ndl_node);
88                 *ndpp = ndl->ndl_node;
89                 return 0;
90         }
91
92         if (!create)
93                 return -ENOENT;
94
95         LIBCFS_ALLOC(*ndpp, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
96         if (*ndpp == NULL)
97                 return -ENOMEM;
98
99         ndl = (lstcon_ndlink_t *)(*ndpp + 1);
100
101         ndl->ndl_node = *ndpp;
102
103         ndl->ndl_node->nd_ref   = 1;
104         ndl->ndl_node->nd_id    = id;
105         ndl->ndl_node->nd_stamp = cfs_time_current();
106         ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
107         ndl->ndl_node->nd_timeout = 0;
108         memset(&ndl->ndl_node->nd_ping, 0, sizeof(lstcon_rpc_t));
109
110         /* queued in global hash & list, no refcount is taken by
111          * global hash & list, if caller release his refcount,
112          * node will be released */
113         list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]);
114         list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list);
115
116         return 0;
117 }
118
119 void
120 lstcon_node_put(lstcon_node_t *nd)
121 {
122         lstcon_ndlink_t *ndl;
123
124         LASSERT(nd->nd_ref > 0);
125
126         if (--nd->nd_ref > 0)
127                 return;
128
129         ndl = (lstcon_ndlink_t *)(nd + 1);
130
131         LASSERT(!list_empty(&ndl->ndl_link));
132         LASSERT(!list_empty(&ndl->ndl_hlink));
133
134         /* remove from session */
135         list_del(&ndl->ndl_link);
136         list_del(&ndl->ndl_hlink);
137
138         LIBCFS_FREE(nd, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
139 }
140
141 static int
142 lstcon_ndlink_find(struct list_head *hash,
143                    lnet_process_id_t id, lstcon_ndlink_t **ndlpp, int create)
144 {
145         unsigned int     idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
146         lstcon_ndlink_t *ndl;
147         lstcon_node_t   *nd;
148         int              rc;
149
150         if (id.nid == LNET_NID_ANY)
151                 return -EINVAL;
152
153         /* search in hash */
154         list_for_each_entry(ndl, &hash[idx], ndl_hlink) {
155                 if (ndl->ndl_node->nd_id.nid != id.nid ||
156                     ndl->ndl_node->nd_id.pid != id.pid)
157                         continue;
158
159                 *ndlpp = ndl;
160                 return 0;
161         }
162
163         if (create == 0)
164                 return -ENOENT;
165
166         /* find or create in session hash */
167         rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0);
168         if (rc != 0)
169                 return rc;
170
171         LIBCFS_ALLOC(ndl, sizeof(lstcon_ndlink_t));
172         if (ndl == NULL) {
173                 lstcon_node_put(nd);
174                 return -ENOMEM;
175         }
176
177         *ndlpp = ndl;
178
179         ndl->ndl_node = nd;
180         INIT_LIST_HEAD(&ndl->ndl_link);
181         list_add_tail(&ndl->ndl_hlink, &hash[idx]);
182
183         return 0;
184 }
185
186 static void
187 lstcon_ndlink_release(lstcon_ndlink_t *ndl)
188 {
189         LASSERT(list_empty(&ndl->ndl_link));
190         LASSERT(!list_empty(&ndl->ndl_hlink));
191
192         list_del(&ndl->ndl_hlink); /* delete from hash */
193         lstcon_node_put(ndl->ndl_node);
194
195         LIBCFS_FREE(ndl, sizeof(*ndl));
196 }
197
198 static int
199 lstcon_group_alloc(char *name, lstcon_group_t **grpp)
200 {
201         lstcon_group_t *grp;
202         int             i;
203
204         LIBCFS_ALLOC(grp, offsetof(lstcon_group_t,
205                                    grp_ndl_hash[LST_NODE_HASHSIZE]));
206         if (grp == NULL)
207                 return -ENOMEM;
208
209         memset(grp, 0, offsetof(lstcon_group_t,
210                                 grp_ndl_hash[LST_NODE_HASHSIZE]));
211
212         grp->grp_ref = 1;
213         if (name != NULL) {
214                 if (strlen(name) > sizeof(grp->grp_name)-1) {
215                         LIBCFS_FREE(grp, offsetof(lstcon_group_t,
216                                           grp_ndl_hash[LST_NODE_HASHSIZE]));
217                         return -E2BIG;
218                 }
219                 strncpy(grp->grp_name, name, sizeof(grp->grp_name));
220         }
221
222         INIT_LIST_HEAD(&grp->grp_link);
223         INIT_LIST_HEAD(&grp->grp_ndl_list);
224         INIT_LIST_HEAD(&grp->grp_trans_list);
225
226         for (i = 0; i < LST_NODE_HASHSIZE; i++)
227                 INIT_LIST_HEAD(&grp->grp_ndl_hash[i]);
228
229         *grpp = grp;
230
231         return 0;
232 }
233
234 static void
235 lstcon_group_addref(lstcon_group_t *grp)
236 {
237         grp->grp_ref++;
238 }
239
240 static void lstcon_group_ndlink_release(lstcon_group_t *, lstcon_ndlink_t *);
241
242 static void
243 lstcon_group_drain(lstcon_group_t *grp, int keep)
244 {
245         lstcon_ndlink_t *ndl;
246         lstcon_ndlink_t *tmp;
247
248         list_for_each_entry_safe(ndl, tmp, &grp->grp_ndl_list, ndl_link) {
249                 if ((ndl->ndl_node->nd_state & keep) == 0)
250                         lstcon_group_ndlink_release(grp, ndl);
251         }
252 }
253
254 static void
255 lstcon_group_decref(lstcon_group_t *grp)
256 {
257         int i;
258
259         if (--grp->grp_ref > 0)
260                 return;
261
262         if (!list_empty(&grp->grp_link))
263                 list_del(&grp->grp_link);
264
265         lstcon_group_drain(grp, 0);
266
267         for (i = 0; i < LST_NODE_HASHSIZE; i++)
268                 LASSERT(list_empty(&grp->grp_ndl_hash[i]));
269
270         LIBCFS_FREE(grp, offsetof(lstcon_group_t,
271                                   grp_ndl_hash[LST_NODE_HASHSIZE]));
272 }
273
274 static int
275 lstcon_group_find(const char *name, lstcon_group_t **grpp)
276 {
277         lstcon_group_t *grp;
278
279         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
280                 if (strncmp(grp->grp_name, name, LST_NAME_SIZE) != 0)
281                         continue;
282
283                 lstcon_group_addref(grp);  /* +1 ref for caller */
284                 *grpp = grp;
285                 return 0;
286         }
287
288         return -ENOENT;
289 }
290
291 static void
292 lstcon_group_put(lstcon_group_t *grp)
293 {
294         lstcon_group_decref(grp);
295 }
296
297 static int
298 lstcon_group_ndlink_find(lstcon_group_t *grp, lnet_process_id_t id,
299                          lstcon_ndlink_t **ndlpp, int create)
300 {
301         int rc;
302
303         rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create);
304         if (rc != 0)
305                 return rc;
306
307         if (!list_empty(&(*ndlpp)->ndl_link))
308                 return 0;
309
310         list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list);
311         grp->grp_nnode++;
312
313         return 0;
314 }
315
316 static void
317 lstcon_group_ndlink_release(lstcon_group_t *grp, lstcon_ndlink_t *ndl)
318 {
319         list_del_init(&ndl->ndl_link);
320         lstcon_ndlink_release(ndl);
321         grp->grp_nnode--;
322 }
323
324 static void
325 lstcon_group_ndlink_move(lstcon_group_t *old,
326                          lstcon_group_t *new, lstcon_ndlink_t *ndl)
327 {
328         unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) %
329                                         LST_NODE_HASHSIZE;
330
331         list_del(&ndl->ndl_hlink);
332         list_del(&ndl->ndl_link);
333         old->grp_nnode--;
334
335         list_add_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]);
336         list_add_tail(&ndl->ndl_link, &new->grp_ndl_list);
337         new->grp_nnode++;
338
339         return;
340 }
341
342 static void
343 lstcon_group_move(lstcon_group_t *old, lstcon_group_t *new)
344 {
345         lstcon_ndlink_t *ndl;
346
347         while (!list_empty(&old->grp_ndl_list)) {
348                 ndl = list_entry(old->grp_ndl_list.next,
349                                  lstcon_ndlink_t, ndl_link);
350                 lstcon_group_ndlink_move(old, new, ndl);
351         }
352 }
353
354 int
355 lstcon_sesrpc_condition(int transop, lstcon_node_t *nd, void *arg)
356 {
357         lstcon_group_t *grp = (lstcon_group_t *)arg;
358
359         switch (transop) {
360         case LST_TRANS_SESNEW:
361                 if (nd->nd_state == LST_NODE_ACTIVE)
362                         return 0;
363                 break;
364
365         case LST_TRANS_SESEND:
366                 if (nd->nd_state != LST_NODE_ACTIVE)
367                         return 0;
368
369                 if (grp != NULL && nd->nd_ref > 1)
370                         return 0;
371                 break;
372
373         case LST_TRANS_SESQRY:
374                 break;
375
376         default:
377                 LBUG();
378         }
379
380         return 1;
381 }
382
383 int
384 lstcon_sesrpc_readent(int transop, srpc_msg_t *msg,
385                       lstcon_rpc_ent_t __user *ent_up)
386 {
387         srpc_debug_reply_t *rep;
388
389         switch (transop) {
390         case LST_TRANS_SESNEW:
391         case LST_TRANS_SESEND:
392                 return 0;
393
394         case LST_TRANS_SESQRY:
395                 rep = &msg->msg_body.dbg_reply;
396
397                 if (copy_to_user(&ent_up->rpe_priv[0],
398                                      &rep->dbg_timeout, sizeof(int)) ||
399                     copy_to_user(&ent_up->rpe_payload[0],
400                                      &rep->dbg_name, LST_NAME_SIZE))
401                         return -EFAULT;
402
403                 return 0;
404
405         default:
406                 LBUG();
407         }
408
409         return 0;
410 }
411
412 static int
413 lstcon_group_nodes_add(lstcon_group_t *grp,
414                        int count, lnet_process_id_t __user *ids_up,
415                        unsigned *featp, struct list_head __user *result_up)
416 {
417         lstcon_rpc_trans_t      *trans;
418         lstcon_ndlink_t         *ndl;
419         lstcon_group_t          *tmp;
420         lnet_process_id_t        id;
421         int                      i;
422         int                      rc;
423
424         rc = lstcon_group_alloc(NULL, &tmp);
425         if (rc != 0) {
426                 CERROR("Out of memory\n");
427                 return -ENOMEM;
428         }
429
430         for (i = 0 ; i < count; i++) {
431                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
432                         rc = -EFAULT;
433                         break;
434                 }
435
436                 /* skip if it's in this group already */
437                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 0);
438                 if (rc == 0)
439                         continue;
440
441                 /* add to tmp group */
442                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1);
443                 if (rc != 0) {
444                         CERROR("Can't create ndlink, out of memory\n");
445                         break;
446                 }
447         }
448
449         if (rc != 0) {
450                 lstcon_group_put(tmp);
451                 return rc;
452         }
453
454         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
455                                      &tmp->grp_trans_list, LST_TRANS_SESNEW,
456                                      tmp, lstcon_sesrpc_condition, &trans);
457         if (rc != 0) {
458                 CERROR("Can't create transaction: %d\n", rc);
459                 lstcon_group_put(tmp);
460                 return rc;
461         }
462
463         /* post all RPCs */
464         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
465
466         rc = lstcon_rpc_trans_interpreter(trans, result_up,
467                                           lstcon_sesrpc_readent);
468         *featp = trans->tas_features;
469
470         /* destroy all RPGs */
471         lstcon_rpc_trans_destroy(trans);
472
473         lstcon_group_move(tmp, grp);
474         lstcon_group_put(tmp);
475
476         return rc;
477 }
478
479 static int
480 lstcon_group_nodes_remove(lstcon_group_t *grp,
481                           int count, lnet_process_id_t __user *ids_up,
482                           struct list_head __user *result_up)
483 {
484         lstcon_rpc_trans_t     *trans;
485         lstcon_ndlink_t        *ndl;
486         lstcon_group_t         *tmp;
487         lnet_process_id_t       id;
488         int                     rc;
489         int                     i;
490
491         /* End session and remove node from the group */
492
493         rc = lstcon_group_alloc(NULL, &tmp);
494         if (rc != 0) {
495                 CERROR("Out of memory\n");
496                 return -ENOMEM;
497         }
498
499         for (i = 0; i < count; i++) {
500                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
501                         rc = -EFAULT;
502                         goto error;
503                 }
504
505                 /* move node to tmp group */
506                 if (lstcon_group_ndlink_find(grp, id, &ndl, 0) == 0)
507                         lstcon_group_ndlink_move(grp, tmp, ndl);
508         }
509
510         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
511                                      &tmp->grp_trans_list, LST_TRANS_SESEND,
512                                      tmp, lstcon_sesrpc_condition, &trans);
513         if (rc != 0) {
514                 CERROR("Can't create transaction: %d\n", rc);
515                 goto error;
516         }
517
518         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
519
520         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
521
522         lstcon_rpc_trans_destroy(trans);
523         /* release nodes anyway, because we can't rollback status */
524         lstcon_group_put(tmp);
525
526         return rc;
527 error:
528         lstcon_group_move(tmp, grp);
529         lstcon_group_put(tmp);
530
531         return rc;
532 }
533
534 int
535 lstcon_group_add(char *name)
536 {
537         lstcon_group_t *grp;
538         int             rc;
539
540         rc = (lstcon_group_find(name, &grp) == 0)? -EEXIST: 0;
541         if (rc != 0) {
542                 /* find a group with same name */
543                 lstcon_group_put(grp);
544                 return rc;
545         }
546
547         rc = lstcon_group_alloc(name, &grp);
548         if (rc != 0) {
549                 CERROR("Can't allocate descriptor for group %s\n", name);
550                 return -ENOMEM;
551         }
552
553         list_add_tail(&grp->grp_link, &console_session.ses_grp_list);
554
555         return rc;
556 }
557
558 int
559 lstcon_nodes_add(char *name, int count, lnet_process_id_t __user *ids_up,
560                  unsigned *featp, struct list_head __user *result_up)
561 {
562         lstcon_group_t         *grp;
563         int                     rc;
564
565         LASSERT (count > 0);
566         LASSERT (ids_up != NULL);
567
568         rc = lstcon_group_find(name, &grp);
569         if (rc != 0) {
570                 CDEBUG(D_NET, "Can't find group %s\n", name);
571                 return rc;
572         }
573
574         if (grp->grp_ref > 2) {
575                 /* referred by other threads or test */
576                 CDEBUG(D_NET, "Group %s is busy\n", name);
577                 lstcon_group_put(grp);
578
579                 return -EBUSY;
580         }
581
582         rc = lstcon_group_nodes_add(grp, count, ids_up, featp, result_up);
583
584         lstcon_group_put(grp);
585
586         return rc;
587 }
588
589 int
590 lstcon_group_del(char *name)
591 {
592         lstcon_rpc_trans_t *trans;
593         lstcon_group_t     *grp;
594         int                 rc;
595
596         rc = lstcon_group_find(name, &grp);
597         if (rc != 0) {
598                 CDEBUG(D_NET, "Can't find group: %s\n", name);
599                 return rc;
600         }
601
602         if (grp->grp_ref > 2) {
603                 /* referred by others threads or test */
604                 CDEBUG(D_NET, "Group %s is busy\n", name);
605                 lstcon_group_put(grp);
606                 return -EBUSY;
607         }
608
609         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
610                                      &grp->grp_trans_list, LST_TRANS_SESEND,
611                                      grp, lstcon_sesrpc_condition, &trans);
612         if (rc != 0) {
613                 CERROR("Can't create transaction: %d\n", rc);
614                 lstcon_group_put(grp);
615                 return rc;
616         }
617
618         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
619
620         lstcon_rpc_trans_destroy(trans);
621
622         lstcon_group_put(grp);
623         /* -ref for session, it's destroyed,
624          * status can't be rolled back, destroy group anway */
625         lstcon_group_put(grp);
626
627         return rc;
628 }
629
630 int
631 lstcon_group_clean(char *name, int args)
632 {
633         lstcon_group_t *grp = NULL;
634         int             rc;
635
636         rc = lstcon_group_find(name, &grp);
637         if (rc != 0) {
638                 CDEBUG(D_NET, "Can't find group %s\n", name);
639                 return rc;
640         }
641
642         if (grp->grp_ref > 2) {
643                 /* referred by test */
644                 CDEBUG(D_NET, "Group %s is busy\n", name);
645                 lstcon_group_put(grp);
646                 return -EBUSY;
647         }
648
649         args = (LST_NODE_ACTIVE | LST_NODE_BUSY |
650                 LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args;
651
652         lstcon_group_drain(grp, args);
653
654         lstcon_group_put(grp);
655         /* release empty group */
656         if (list_empty(&grp->grp_ndl_list))
657                 lstcon_group_put(grp);
658
659         return 0;
660 }
661
662 int
663 lstcon_nodes_remove(char *name, int count,
664                     lnet_process_id_t __user *ids_up,
665                     struct list_head __user *result_up)
666 {
667         lstcon_group_t *grp = NULL;
668         int             rc;
669
670         rc = lstcon_group_find(name, &grp);
671         if (rc != 0) {
672                 CDEBUG(D_NET, "Can't find group: %s\n", name);
673                 return rc;
674         }
675
676         if (grp->grp_ref > 2) {
677                 /* referred by test */
678                 CDEBUG(D_NET, "Group %s is busy\n", name);
679                 lstcon_group_put(grp);
680                 return -EBUSY;
681         }
682
683         rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up);
684
685         lstcon_group_put(grp);
686         /* release empty group */
687         if (list_empty(&grp->grp_ndl_list))
688                 lstcon_group_put(grp);
689
690         return rc;
691 }
692
693 int
694 lstcon_group_refresh(char *name, struct list_head __user *result_up)
695 {
696         lstcon_rpc_trans_t      *trans;
697         lstcon_group_t          *grp;
698         int                      rc;
699
700         rc = lstcon_group_find(name, &grp);
701         if (rc != 0) {
702                 CDEBUG(D_NET, "Can't find group: %s\n", name);
703                 return rc;
704         }
705
706         if (grp->grp_ref > 2) {
707                 /* referred by test */
708                 CDEBUG(D_NET, "Group %s is busy\n", name);
709                 lstcon_group_put(grp);
710                 return -EBUSY;
711         }
712
713         /* re-invite all inactive nodes int the group */
714         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
715                                      &grp->grp_trans_list, LST_TRANS_SESNEW,
716                                      grp, lstcon_sesrpc_condition, &trans);
717         if (rc != 0) {
718                 /* local error, return */
719                 CDEBUG(D_NET, "Can't create transaction: %d\n", rc);
720                 lstcon_group_put(grp);
721                 return rc;
722         }
723
724         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
725
726         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
727
728         lstcon_rpc_trans_destroy(trans);
729         /* -ref for me */
730         lstcon_group_put(grp);
731
732         return rc;
733 }
734
735 int
736 lstcon_group_list(int index, int len, char __user *name_up)
737 {
738         lstcon_group_t *grp;
739
740         LASSERT(index >= 0);
741         LASSERT(name_up != NULL);
742
743         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
744                 if (index-- == 0) {
745                         return copy_to_user(name_up, grp->grp_name, len) ?
746                                             -EFAULT : 0;
747                 }
748         }
749
750         return -ENOENT;
751 }
752
753 static int
754 lstcon_nodes_getent(struct list_head *head, int *index_p,
755                     int *count_p, lstcon_node_ent_t __user *dents_up)
756 {
757         lstcon_ndlink_t  *ndl;
758         lstcon_node_t    *nd;
759         int               count = 0;
760         int               index = 0;
761
762         LASSERT(index_p != NULL && count_p != NULL);
763         LASSERT(dents_up != NULL);
764         LASSERT(*index_p >= 0);
765         LASSERT(*count_p > 0);
766
767         list_for_each_entry(ndl, head, ndl_link) {
768                 if (index++ < *index_p)
769                         continue;
770
771                 if (count >= *count_p)
772                         break;
773
774                 nd = ndl->ndl_node;
775                 if (copy_to_user(&dents_up[count].nde_id,
776                                      &nd->nd_id, sizeof(nd->nd_id)) ||
777                     copy_to_user(&dents_up[count].nde_state,
778                                      &nd->nd_state, sizeof(nd->nd_state)))
779                         return -EFAULT;
780
781                 count ++;
782         }
783
784         if (index <= *index_p)
785                 return -ENOENT;
786
787         *count_p = count;
788         *index_p = index;
789
790         return 0;
791 }
792
793 int
794 lstcon_group_info(char *name, lstcon_ndlist_ent_t __user *gents_p,
795                   int *index_p, int *count_p,
796                   lstcon_node_ent_t __user *dents_up)
797 {
798         lstcon_ndlist_ent_t *gentp;
799         lstcon_group_t      *grp;
800         lstcon_ndlink_t     *ndl;
801         int                  rc;
802
803         rc = lstcon_group_find(name, &grp);
804         if (rc != 0) {
805                 CDEBUG(D_NET, "Can't find group %s\n", name);
806                 return rc;
807         }
808
809         if (dents_up != 0) {
810                 /* verbose query */
811                 rc = lstcon_nodes_getent(&grp->grp_ndl_list,
812                                          index_p, count_p, dents_up);
813                 lstcon_group_put(grp);
814
815                 return rc;
816         }
817
818         /* non-verbose query */
819         LIBCFS_ALLOC(gentp, sizeof(lstcon_ndlist_ent_t));
820         if (gentp == NULL) {
821                 CERROR("Can't allocate ndlist_ent\n");
822                 lstcon_group_put(grp);
823
824                 return -ENOMEM;
825         }
826
827         memset(gentp, 0, sizeof(lstcon_ndlist_ent_t));
828
829         list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link)
830                 LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp);
831
832         rc = copy_to_user(gents_p, gentp,
833                           sizeof(lstcon_ndlist_ent_t)) ? -EFAULT : 0;
834
835         LIBCFS_FREE(gentp, sizeof(lstcon_ndlist_ent_t));
836
837         lstcon_group_put(grp);
838
839         return 0;
840 }
841
842 static int
843 lstcon_batch_find(const char *name, lstcon_batch_t **batpp)
844 {
845         lstcon_batch_t *bat;
846
847         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
848                 if (strncmp(bat->bat_name, name, LST_NAME_SIZE) == 0) {
849                         *batpp = bat;
850                         return 0;
851                 }
852         }
853
854         return -ENOENT;
855 }
856
857 int
858 lstcon_batch_add(char *name)
859 {
860         lstcon_batch_t   *bat;
861         int               i;
862         int               rc;
863
864         rc = (lstcon_batch_find(name, &bat) == 0)? -EEXIST: 0;
865         if (rc != 0) {
866                 CDEBUG(D_NET, "Batch %s already exists\n", name);
867                 return rc;
868         }
869
870         LIBCFS_ALLOC(bat, sizeof(lstcon_batch_t));
871         if (bat == NULL) {
872                 CERROR("Can't allocate descriptor for batch %s\n", name);
873                 return -ENOMEM;
874         }
875
876         LIBCFS_ALLOC(bat->bat_cli_hash,
877                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
878         if (bat->bat_cli_hash == NULL) {
879                 CERROR("Can't allocate hash for batch %s\n", name);
880                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
881
882                 return -ENOMEM;
883         }
884
885         LIBCFS_ALLOC(bat->bat_srv_hash,
886                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
887         if (bat->bat_srv_hash == NULL) {
888                 CERROR("Can't allocate hash for batch %s\n", name);
889                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
890                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
891
892                 return -ENOMEM;
893         }
894
895         if (strlen(name) > sizeof(bat->bat_name)-1) {
896                 LIBCFS_FREE(bat->bat_srv_hash, LST_NODE_HASHSIZE);
897                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
898                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
899                 return -E2BIG;
900         }
901         strncpy(bat->bat_name, name, sizeof(bat->bat_name));
902         bat->bat_hdr.tsb_index = 0;
903         bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie;
904
905         bat->bat_ntest = 0;
906         bat->bat_state = LST_BATCH_IDLE;
907
908         INIT_LIST_HEAD(&bat->bat_cli_list);
909         INIT_LIST_HEAD(&bat->bat_srv_list);
910         INIT_LIST_HEAD(&bat->bat_test_list);
911         INIT_LIST_HEAD(&bat->bat_trans_list);
912
913         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
914                 INIT_LIST_HEAD(&bat->bat_cli_hash[i]);
915                 INIT_LIST_HEAD(&bat->bat_srv_hash[i]);
916         }
917
918         list_add_tail(&bat->bat_link, &console_session.ses_bat_list);
919
920         return rc;
921 }
922
923 int
924 lstcon_batch_list(int index, int len, char __user *name_up)
925 {
926         lstcon_batch_t *bat;
927
928         LASSERT(name_up != NULL);
929         LASSERT(index >= 0);
930
931         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
932                 if (index-- == 0) {
933                         return copy_to_user(name_up, bat->bat_name, len) ?
934                                             -EFAULT : 0;
935                 }
936         }
937
938         return -ENOENT;
939 }
940
941 int
942 lstcon_batch_info(char *name, lstcon_test_batch_ent_t __user *ent_up,
943                   int server, int testidx, int *index_p, int *ndent_p,
944                   lstcon_node_ent_t __user *dents_up)
945 {
946         lstcon_test_batch_ent_t *entp;
947         struct list_head        *clilst;
948         struct list_head        *srvlst;
949         lstcon_test_t           *test = NULL;
950         lstcon_batch_t          *bat;
951         lstcon_ndlink_t         *ndl;
952         int                      rc;
953
954         rc = lstcon_batch_find(name, &bat);
955         if (rc != 0) {
956                 CDEBUG(D_NET, "Can't find batch %s\n", name);
957                 return -ENOENT;
958         }
959
960         if (testidx > 0) {
961                 /* query test, test index start from 1 */
962                 list_for_each_entry(test, &bat->bat_test_list, tes_link) {
963                         if (testidx-- == 1)
964                                 break;
965                 }
966
967                 if (testidx > 0) {
968                         CDEBUG(D_NET, "Can't find specified test in batch\n");
969                         return -ENOENT;
970                 }
971         }
972
973         clilst = (test == NULL) ? &bat->bat_cli_list :
974                                   &test->tes_src_grp->grp_ndl_list;
975         srvlst = (test == NULL) ? &bat->bat_srv_list :
976                                   &test->tes_dst_grp->grp_ndl_list;
977
978         if (dents_up != NULL) {
979                 rc = lstcon_nodes_getent((server ? srvlst: clilst),
980                                          index_p, ndent_p, dents_up);
981                 return rc;
982         }
983
984         /* non-verbose query */
985         LIBCFS_ALLOC(entp, sizeof(lstcon_test_batch_ent_t));
986         if (entp == NULL)
987                 return -ENOMEM;
988
989         memset(entp, 0, sizeof(lstcon_test_batch_ent_t));
990
991         if (test == NULL) {
992                 entp->u.tbe_batch.bae_ntest = bat->bat_ntest;
993                 entp->u.tbe_batch.bae_state = bat->bat_state;
994
995         } else {
996
997                 entp->u.tbe_test.tse_type   = test->tes_type;
998                 entp->u.tbe_test.tse_loop   = test->tes_loop;
999                 entp->u.tbe_test.tse_concur = test->tes_concur;
1000         }
1001
1002         list_for_each_entry(ndl, clilst, ndl_link)
1003                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle);
1004
1005         list_for_each_entry(ndl, srvlst, ndl_link)
1006                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle);
1007
1008         rc = copy_to_user(ent_up, entp,
1009                           sizeof(lstcon_test_batch_ent_t)) ? -EFAULT : 0;
1010
1011         LIBCFS_FREE(entp, sizeof(lstcon_test_batch_ent_t));
1012
1013         return rc;
1014 }
1015
1016 int
1017 lstcon_batrpc_condition(int transop, lstcon_node_t *nd, void *arg)
1018 {
1019         switch (transop) {
1020         case LST_TRANS_TSBRUN:
1021                 if (nd->nd_state != LST_NODE_ACTIVE)
1022                         return -ENETDOWN;
1023                 break;
1024
1025         case LST_TRANS_TSBSTOP:
1026                 if (nd->nd_state != LST_NODE_ACTIVE)
1027                         return 0;
1028                 break;
1029
1030         case LST_TRANS_TSBCLIQRY:
1031         case LST_TRANS_TSBSRVQRY:
1032                 break;
1033         }
1034
1035         return 1;
1036 }
1037
1038 static int
1039 lstcon_batch_op(lstcon_batch_t *bat, int transop,
1040                 struct list_head __user *result_up)
1041 {
1042         lstcon_rpc_trans_t *trans;
1043         int                 rc;
1044
1045         rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list,
1046                                      &bat->bat_trans_list, transop,
1047                                      bat, lstcon_batrpc_condition, &trans);
1048         if (rc != 0) {
1049                 CERROR("Can't create transaction: %d\n", rc);
1050                 return rc;
1051         }
1052
1053         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1054
1055         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1056
1057         lstcon_rpc_trans_destroy(trans);
1058
1059         return rc;
1060 }
1061
1062 int
1063 lstcon_batch_run(char *name, int timeout, struct list_head __user *result_up)
1064 {
1065         lstcon_batch_t *bat;
1066         int             rc;
1067
1068         if (lstcon_batch_find(name, &bat) != 0) {
1069                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1070                 return -ENOENT;
1071         }
1072
1073         bat->bat_arg = timeout;
1074
1075         rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up);
1076
1077         /* mark batch as running if it's started in any node */
1078         if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0) != 0)
1079                 bat->bat_state = LST_BATCH_RUNNING;
1080
1081         return rc;
1082 }
1083
1084 int
1085 lstcon_batch_stop(char *name, int force, struct list_head __user *result_up)
1086 {
1087         lstcon_batch_t *bat;
1088         int             rc;
1089
1090         if (lstcon_batch_find(name, &bat) != 0) {
1091                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1092                 return -ENOENT;
1093         }
1094
1095         bat->bat_arg = force;
1096
1097         rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up);
1098
1099         /* mark batch as stopped if all RPCs finished */
1100         if (lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0) == 0)
1101                 bat->bat_state = LST_BATCH_IDLE;
1102
1103         return rc;
1104 }
1105
1106 static void
1107 lstcon_batch_destroy(lstcon_batch_t *bat)
1108 {
1109         lstcon_ndlink_t    *ndl;
1110         lstcon_test_t      *test;
1111         int                 i;
1112
1113         list_del(&bat->bat_link);
1114
1115         while (!list_empty(&bat->bat_test_list)) {
1116                 test = list_entry(bat->bat_test_list.next,
1117                                   lstcon_test_t, tes_link);
1118                 LASSERT(list_empty(&test->tes_trans_list));
1119
1120                 list_del(&test->tes_link);
1121
1122                 lstcon_group_put(test->tes_src_grp);
1123                 lstcon_group_put(test->tes_dst_grp);
1124
1125                 LIBCFS_FREE(test, offsetof(lstcon_test_t,
1126                                            tes_param[test->tes_paramlen]));
1127         }
1128
1129         LASSERT(list_empty(&bat->bat_trans_list));
1130
1131         while (!list_empty(&bat->bat_cli_list)) {
1132                 ndl = list_entry(bat->bat_cli_list.next,
1133                                  lstcon_ndlink_t, ndl_link);
1134                 list_del_init(&ndl->ndl_link);
1135
1136                 lstcon_ndlink_release(ndl);
1137         }
1138
1139         while (!list_empty(&bat->bat_srv_list)) {
1140                 ndl = list_entry(bat->bat_srv_list.next,
1141                                  lstcon_ndlink_t, ndl_link);
1142                 list_del_init(&ndl->ndl_link);
1143
1144                 lstcon_ndlink_release(ndl);
1145         }
1146
1147         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1148                 LASSERT(list_empty(&bat->bat_cli_hash[i]));
1149                 LASSERT(list_empty(&bat->bat_srv_hash[i]));
1150         }
1151
1152         LIBCFS_FREE(bat->bat_cli_hash,
1153                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1154         LIBCFS_FREE(bat->bat_srv_hash,
1155                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1156         LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
1157 }
1158
1159 int
1160 lstcon_testrpc_condition(int transop, lstcon_node_t *nd, void *arg)
1161 {
1162         lstcon_test_t    *test;
1163         lstcon_batch_t   *batch;
1164         lstcon_ndlink_t  *ndl;
1165         struct list_head *hash;
1166         struct list_head *head;
1167
1168         test = (lstcon_test_t *)arg;
1169         LASSERT(test != NULL);
1170
1171         batch = test->tes_batch;
1172         LASSERT(batch != NULL);
1173
1174         if (test->tes_oneside &&
1175             transop == LST_TRANS_TSBSRVADD)
1176                 return 0;
1177
1178         if (nd->nd_state != LST_NODE_ACTIVE)
1179                 return -ENETDOWN;
1180
1181         if (transop == LST_TRANS_TSBCLIADD) {
1182                 hash = batch->bat_cli_hash;
1183                 head = &batch->bat_cli_list;
1184
1185         } else {
1186                 LASSERT (transop == LST_TRANS_TSBSRVADD);
1187
1188                 hash = batch->bat_srv_hash;
1189                 head = &batch->bat_srv_list;
1190         }
1191
1192         LASSERT (nd->nd_id.nid != LNET_NID_ANY);
1193
1194         if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1) != 0)
1195                 return -ENOMEM;
1196
1197         if (list_empty(&ndl->ndl_link))
1198                 list_add_tail(&ndl->ndl_link, head);
1199
1200         return 1;
1201 }
1202
1203 static int
1204 lstcon_test_nodes_add(lstcon_test_t *test, struct list_head __user *result_up)
1205 {
1206         lstcon_rpc_trans_t     *trans;
1207         lstcon_group_t         *grp;
1208         int                     transop;
1209         int                     rc;
1210
1211         LASSERT (test->tes_src_grp != NULL);
1212         LASSERT (test->tes_dst_grp != NULL);
1213
1214         transop = LST_TRANS_TSBSRVADD;
1215         grp  = test->tes_dst_grp;
1216 again:
1217         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
1218                                      &test->tes_trans_list, transop,
1219                                      test, lstcon_testrpc_condition, &trans);
1220         if (rc != 0) {
1221                 CERROR("Can't create transaction: %d\n", rc);
1222                 return rc;
1223         }
1224
1225         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1226
1227         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1228             lstcon_trans_stat()->trs_fwk_errno != 0) {
1229                 lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1230
1231                 lstcon_rpc_trans_destroy(trans);
1232                 /* return if any error */
1233                 CDEBUG(D_NET, "Failed to add test %s, "
1234                               "RPC error %d, framework error %d\n",
1235                        transop == LST_TRANS_TSBCLIADD ? "client" : "server",
1236                        lstcon_trans_stat()->trs_rpc_errno,
1237                        lstcon_trans_stat()->trs_fwk_errno);
1238
1239                 return rc;
1240         }
1241
1242         lstcon_rpc_trans_destroy(trans);
1243
1244         if (transop == LST_TRANS_TSBCLIADD)
1245                 return rc;
1246
1247         transop = LST_TRANS_TSBCLIADD;
1248         grp = test->tes_src_grp;
1249         test->tes_cliidx = 0;
1250
1251         /* requests to test clients */
1252         goto again;
1253 }
1254
1255 static int
1256 lstcon_verify_batch(const char *name, lstcon_batch_t **batch)
1257 {
1258         int rc;
1259
1260         rc = lstcon_batch_find(name, batch);
1261         if (rc != 0) {
1262                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1263                 return rc;
1264         }
1265
1266         if ((*batch)->bat_state != LST_BATCH_IDLE) {
1267                 CDEBUG(D_NET, "Can't change running batch %s\n", name);
1268                 return -EINVAL;
1269         }
1270
1271         return 0;
1272 }
1273
1274 static int
1275 lstcon_verify_group(const char *name, lstcon_group_t **grp)
1276 {
1277         int                     rc;
1278         lstcon_ndlink_t         *ndl;
1279
1280         rc = lstcon_group_find(name, grp);
1281         if (rc != 0) {
1282                 CDEBUG(D_NET, "can't find group %s\n", name);
1283                 return rc;
1284         }
1285
1286         list_for_each_entry(ndl, &(*grp)->grp_ndl_list, ndl_link) {
1287                 if (ndl->ndl_node->nd_state == LST_NODE_ACTIVE) {
1288                         return 0;
1289                 }
1290         }
1291
1292         CDEBUG(D_NET, "Group %s has no ACTIVE nodes\n", name);
1293
1294         return -EINVAL;
1295 }
1296
1297 int
1298 lstcon_test_add(char *batch_name, int type, int loop,
1299                 int concur, int dist, int span,
1300                 char *src_name, char *dst_name,
1301                 void *param, int paramlen, int *retp,
1302                 struct list_head __user *result_up)
1303 {
1304         lstcon_test_t    *test   = NULL;
1305         int              rc;
1306         lstcon_group_t   *src_grp = NULL;
1307         lstcon_group_t   *dst_grp = NULL;
1308         lstcon_batch_t   *batch = NULL;
1309
1310         /*
1311          * verify that a batch of the given name exists, and the groups
1312          * that will be part of the batch exist and have at least one
1313          * active node
1314          */
1315         rc = lstcon_verify_batch(batch_name, &batch);
1316         if (rc != 0)
1317                 goto out;
1318
1319         rc = lstcon_verify_group(src_name, &src_grp);
1320         if (rc != 0)
1321                 goto out;
1322
1323         rc = lstcon_verify_group(dst_name, &dst_grp);
1324         if (rc != 0)
1325                 goto out;
1326
1327         if (dst_grp->grp_userland)
1328                 *retp = 1;
1329
1330         LIBCFS_ALLOC(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1331         if (!test) {
1332                 CERROR("Can't allocate test descriptor\n");
1333                 rc = -ENOMEM;
1334
1335                 goto out;
1336         }
1337
1338         memset(test, 0, offsetof(lstcon_test_t, tes_param[paramlen]));
1339         test->tes_hdr.tsb_id    = batch->bat_hdr.tsb_id;
1340         test->tes_batch         = batch;
1341         test->tes_type          = type;
1342         test->tes_oneside       = 0; /* TODO */
1343         test->tes_loop          = loop;
1344         test->tes_concur        = concur;
1345         test->tes_stop_onerr    = 1; /* TODO */
1346         test->tes_span          = span;
1347         test->tes_dist          = dist;
1348         test->tes_cliidx        = 0; /* just used for creating RPC */
1349         test->tes_src_grp       = src_grp;
1350         test->tes_dst_grp       = dst_grp;
1351         INIT_LIST_HEAD(&test->tes_trans_list);
1352
1353         if (param != NULL) {
1354                 test->tes_paramlen = paramlen;
1355                 memcpy(&test->tes_param[0], param, paramlen);
1356         }
1357
1358         rc = lstcon_test_nodes_add(test, result_up);
1359
1360         if (rc != 0)
1361                 goto out;
1362
1363         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1364             lstcon_trans_stat()->trs_fwk_errno != 0)
1365                 CDEBUG(D_NET, "Failed to add test %d to batch %s\n", type,
1366                        batch_name);
1367
1368         /* add to test list anyway, so user can check what's going on */
1369         list_add_tail(&test->tes_link, &batch->bat_test_list);
1370
1371         batch->bat_ntest++;
1372         test->tes_hdr.tsb_index = batch->bat_ntest;
1373
1374         /*  hold groups so nobody can change them */
1375         return rc;
1376 out:
1377         if (test != NULL)
1378                 LIBCFS_FREE(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1379
1380         if (dst_grp != NULL)
1381                 lstcon_group_put(dst_grp);
1382
1383         if (src_grp != NULL)
1384                 lstcon_group_put(src_grp);
1385
1386         return rc;
1387 }
1388
1389 int
1390 lstcon_test_find(lstcon_batch_t *batch, int idx, lstcon_test_t **testpp)
1391 {
1392         lstcon_test_t *test;
1393
1394         list_for_each_entry(test, &batch->bat_test_list, tes_link) {
1395                 if (idx == test->tes_hdr.tsb_index) {
1396                         *testpp = test;
1397                         return 0;
1398                 }
1399         }
1400
1401         return -ENOENT;
1402 }
1403
1404 int
1405 lstcon_tsbrpc_readent(int transop, srpc_msg_t *msg,
1406                       lstcon_rpc_ent_t __user *ent_up)
1407 {
1408         srpc_batch_reply_t *rep = &msg->msg_body.bat_reply;
1409
1410         LASSERT (transop == LST_TRANS_TSBCLIQRY ||
1411                  transop == LST_TRANS_TSBSRVQRY);
1412
1413         /* positive errno, framework error code */
1414         if (copy_to_user(&ent_up->rpe_priv[0],
1415                              &rep->bar_active, sizeof(rep->bar_active)))
1416                 return -EFAULT;
1417
1418         return 0;
1419 }
1420
1421 int
1422 lstcon_test_batch_query(char *name, int testidx, int client,
1423                         int timeout, struct list_head __user *result_up)
1424 {
1425         lstcon_rpc_trans_t *trans;
1426         struct list_head   *translist;
1427         struct list_head   *ndlist;
1428         lstcon_tsb_hdr_t   *hdr;
1429         lstcon_batch_t     *batch;
1430         lstcon_test_t      *test = NULL;
1431         int                 transop;
1432         int                 rc;
1433
1434         rc = lstcon_batch_find(name, &batch);
1435         if (rc != 0) {
1436                 CDEBUG(D_NET, "Can't find batch: %s\n", name);
1437                 return rc;
1438         }
1439
1440         if (testidx == 0) {
1441                 translist = &batch->bat_trans_list;
1442                 ndlist    = &batch->bat_cli_list;
1443                 hdr       = &batch->bat_hdr;
1444
1445         } else {
1446                 /* query specified test only */
1447                 rc = lstcon_test_find(batch, testidx, &test);
1448                 if (rc != 0) {
1449                         CDEBUG(D_NET, "Can't find test: %d\n", testidx);
1450                         return rc;
1451                 }
1452
1453                 translist = &test->tes_trans_list;
1454                 ndlist    = &test->tes_src_grp->grp_ndl_list;
1455                 hdr       = &test->tes_hdr;
1456         }
1457
1458         transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY;
1459
1460         rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr,
1461                                      lstcon_batrpc_condition, &trans);
1462         if (rc != 0) {
1463                 CERROR("Can't create transaction: %d\n", rc);
1464                 return rc;
1465         }
1466
1467         lstcon_rpc_trans_postwait(trans, timeout);
1468
1469         if (testidx == 0 && /* query a batch, not a test */
1470             lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) == 0 &&
1471             lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0) == 0) {
1472                 /* all RPCs finished, and no active test */
1473                 batch->bat_state = LST_BATCH_IDLE;
1474         }
1475
1476         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1477                                           lstcon_tsbrpc_readent);
1478         lstcon_rpc_trans_destroy(trans);
1479
1480         return rc;
1481 }
1482
1483 int
1484 lstcon_statrpc_readent(int transop, srpc_msg_t *msg,
1485                        lstcon_rpc_ent_t __user *ent_up)
1486 {
1487         srpc_stat_reply_t *rep = &msg->msg_body.stat_reply;
1488         sfw_counters_t __user  *sfwk_stat;
1489         srpc_counters_t __user *srpc_stat;
1490         lnet_counters_t __user *lnet_stat;
1491
1492         if (rep->str_status != 0)
1493                 return 0;
1494
1495         sfwk_stat = (sfw_counters_t __user *)&ent_up->rpe_payload[0];
1496         srpc_stat = (srpc_counters_t __user *)
1497                 ((char __user *)sfwk_stat + sizeof(*sfwk_stat));
1498         lnet_stat = (lnet_counters_t __user *)
1499                 ((char __user *)srpc_stat + sizeof(*srpc_stat));
1500
1501         if (copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) ||
1502             copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) ||
1503             copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat)))
1504                 return -EFAULT;
1505
1506         return 0;
1507 }
1508
1509 int
1510 lstcon_ndlist_stat(struct list_head *ndlist,
1511                    int timeout, struct list_head __user *result_up)
1512 {
1513         struct list_head    head;
1514         lstcon_rpc_trans_t *trans;
1515         int                 rc;
1516
1517         INIT_LIST_HEAD(&head);
1518
1519         rc = lstcon_rpc_trans_ndlist(ndlist, &head,
1520                                      LST_TRANS_STATQRY, NULL, NULL, &trans);
1521         if (rc != 0) {
1522                 CERROR("Can't create transaction: %d\n", rc);
1523                 return rc;
1524         }
1525
1526         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1527
1528         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1529                                           lstcon_statrpc_readent);
1530         lstcon_rpc_trans_destroy(trans);
1531
1532         return rc;
1533 }
1534
1535 int
1536 lstcon_group_stat(char *grp_name, int timeout,
1537                   struct list_head __user *result_up)
1538 {
1539         lstcon_group_t     *grp;
1540         int                 rc;
1541
1542         rc = lstcon_group_find(grp_name, &grp);
1543         if (rc != 0) {
1544                 CDEBUG(D_NET, "Can't find group %s\n", grp_name);
1545                 return rc;
1546         }
1547
1548         rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up);
1549
1550         lstcon_group_put(grp);
1551
1552         return rc;
1553 }
1554
1555 int
1556 lstcon_nodes_stat(int count, lnet_process_id_t __user *ids_up,
1557                   int timeout, struct list_head __user *result_up)
1558 {
1559         lstcon_ndlink_t         *ndl;
1560         lstcon_group_t          *tmp;
1561         lnet_process_id_t        id;
1562         int                      i;
1563         int                      rc;
1564
1565         rc = lstcon_group_alloc(NULL, &tmp);
1566         if (rc != 0) {
1567                 CERROR("Out of memory\n");
1568                 return -ENOMEM;
1569         }
1570
1571         for (i = 0 ; i < count; i++) {
1572                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1573                         rc = -EFAULT;
1574                         break;
1575                 }
1576
1577                 /* add to tmp group */
1578                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2);
1579                 if (rc != 0) {
1580                         CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET,
1581                                "Failed to find or create %s: %d\n",
1582                                libcfs_id2str(id), rc);
1583                         break;
1584                 }
1585         }
1586
1587         if (rc != 0) {
1588                 lstcon_group_put(tmp);
1589                 return rc;
1590         }
1591
1592         rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up);
1593
1594         lstcon_group_put(tmp);
1595
1596         return rc;
1597 }
1598
1599 int
1600 lstcon_debug_ndlist(struct list_head *ndlist,
1601                     struct list_head *translist,
1602                     int timeout, struct list_head __user *result_up)
1603 {
1604         lstcon_rpc_trans_t *trans;
1605         int                 rc;
1606
1607         rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY,
1608                                      NULL, lstcon_sesrpc_condition, &trans);
1609         if (rc != 0) {
1610                 CERROR("Can't create transaction: %d\n", rc);
1611                 return rc;
1612         }
1613
1614         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1615
1616         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1617                                           lstcon_sesrpc_readent);
1618         lstcon_rpc_trans_destroy(trans);
1619
1620         return rc;
1621 }
1622
1623 int
1624 lstcon_session_debug(int timeout, struct list_head __user *result_up)
1625 {
1626         return lstcon_debug_ndlist(&console_session.ses_ndl_list,
1627                                    NULL, timeout, result_up);
1628 }
1629
1630 int
1631 lstcon_batch_debug(int timeout, char *name,
1632                    int client, struct list_head __user *result_up)
1633 {
1634         lstcon_batch_t *bat;
1635         int             rc;
1636
1637         rc = lstcon_batch_find(name, &bat);
1638         if (rc != 0)
1639                 return -ENOENT;
1640
1641         rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list :
1642                                           &bat->bat_srv_list,
1643                                  NULL, timeout, result_up);
1644
1645         return rc;
1646 }
1647
1648 int
1649 lstcon_group_debug(int timeout, char *name,
1650                    struct list_head __user *result_up)
1651 {
1652         lstcon_group_t *grp;
1653         int             rc;
1654
1655         rc = lstcon_group_find(name, &grp);
1656         if (rc != 0)
1657                 return -ENOENT;
1658
1659         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1660                                  timeout, result_up);
1661         lstcon_group_put(grp);
1662
1663         return rc;
1664 }
1665
1666 int
1667 lstcon_nodes_debug(int timeout,
1668                    int count, lnet_process_id_t __user *ids_up,
1669                    struct list_head __user *result_up)
1670 {
1671         lnet_process_id_t  id;
1672         lstcon_ndlink_t   *ndl;
1673         lstcon_group_t    *grp;
1674         int                i;
1675         int                rc;
1676
1677         rc = lstcon_group_alloc(NULL, &grp);
1678         if (rc != 0) {
1679                 CDEBUG(D_NET, "Out of memory\n");
1680                 return rc;
1681         }
1682
1683         for (i = 0; i < count; i++) {
1684                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1685                         rc = -EFAULT;
1686                         break;
1687                 }
1688
1689                 /* node is added to tmp group */
1690                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 1);
1691                 if (rc != 0) {
1692                         CERROR("Can't create node link\n");
1693                         break;
1694                 }
1695         }
1696
1697         if (rc != 0) {
1698                 lstcon_group_put(grp);
1699                 return rc;
1700         }
1701
1702         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1703                                  timeout, result_up);
1704
1705         lstcon_group_put(grp);
1706
1707         return rc;
1708 }
1709
1710 int
1711 lstcon_session_match(lst_sid_t sid)
1712 {
1713         return (console_session.ses_id.ses_nid   == sid.ses_nid &&
1714                 console_session.ses_id.ses_stamp == sid.ses_stamp) ?  1: 0;
1715 }
1716
1717 static void
1718 lstcon_new_session_id(lst_sid_t *sid)
1719 {
1720         lnet_process_id_t      id;
1721
1722         LASSERT (console_session.ses_state == LST_SESSION_NONE);
1723
1724         LNetGetId(1, &id);
1725         sid->ses_nid   = id.nid;
1726         sid->ses_stamp = cfs_time_current();
1727 }
1728
1729 extern srpc_service_t lstcon_acceptor_service;
1730
1731 int
1732 lstcon_session_new(char *name, int key, unsigned feats,
1733                    int timeout, int force, lst_sid_t __user *sid_up)
1734 {
1735         int     rc = 0;
1736         int     i;
1737
1738         if (console_session.ses_state != LST_SESSION_NONE) {
1739                 /* session exists */
1740                 if (!force) {
1741                         CNETERR("Session %s already exists\n",
1742                                 console_session.ses_name);
1743                         return -EEXIST;
1744                 }
1745
1746                 rc = lstcon_session_end();
1747
1748                 /* lstcon_session_end() only return local error */
1749                 if  (rc != 0)
1750                         return rc;
1751         }
1752
1753         if ((feats & ~LST_FEATS_MASK) != 0) {
1754                 CNETERR("Unknown session features %x\n",
1755                         (feats & ~LST_FEATS_MASK));
1756                 return -EINVAL;
1757         }
1758
1759         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
1760                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
1761
1762         lstcon_new_session_id(&console_session.ses_id);
1763
1764         console_session.ses_key     = key;
1765         console_session.ses_state   = LST_SESSION_ACTIVE;
1766         console_session.ses_force   = !!force;
1767         console_session.ses_features = feats;
1768         console_session.ses_feats_updated = 0;
1769         console_session.ses_timeout = (timeout <= 0) ?
1770                                       LST_CONSOLE_TIMEOUT : timeout;
1771
1772         if (strlen(name) > sizeof(console_session.ses_name)-1)
1773                 return -E2BIG;
1774         strlcpy(console_session.ses_name, name,
1775                 sizeof(console_session.ses_name));
1776
1777         rc = lstcon_batch_add(LST_DEFAULT_BATCH);
1778         if (rc != 0)
1779                 return rc;
1780
1781         rc = lstcon_rpc_pinger_start();
1782         if (rc != 0) {
1783                 lstcon_batch_t *bat = NULL;
1784
1785                 lstcon_batch_find(LST_DEFAULT_BATCH, &bat);
1786                 lstcon_batch_destroy(bat);
1787
1788                 return rc;
1789         }
1790
1791         if (copy_to_user(sid_up, &console_session.ses_id,
1792                              sizeof(lst_sid_t)) == 0)
1793                 return rc;
1794
1795         lstcon_session_end();
1796
1797         return -EFAULT;
1798 }
1799
1800 int
1801 lstcon_session_info(lst_sid_t __user *sid_up, int __user *key_up,
1802                     unsigned __user *featp,
1803                     lstcon_ndlist_ent_t __user *ndinfo_up,
1804                     char __user *name_up, int len)
1805 {
1806         lstcon_ndlist_ent_t *entp;
1807         lstcon_ndlink_t     *ndl;
1808         int                  rc = 0;
1809
1810         if (console_session.ses_state != LST_SESSION_ACTIVE)
1811                 return -ESRCH;
1812
1813         LIBCFS_ALLOC(entp, sizeof(*entp));
1814         if (entp == NULL)
1815                 return -ENOMEM;
1816
1817         memset(entp, 0, sizeof(*entp));
1818
1819         list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link)
1820                 LST_NODE_STATE_COUNTER(ndl->ndl_node, entp);
1821
1822         if (copy_to_user(sid_up, &console_session.ses_id,
1823                              sizeof(lst_sid_t)) ||
1824             copy_to_user(key_up, &console_session.ses_key,
1825                              sizeof(*key_up)) ||
1826             copy_to_user(featp, &console_session.ses_features,
1827                              sizeof(*featp)) ||
1828             copy_to_user(ndinfo_up, entp, sizeof(*entp)) ||
1829             copy_to_user(name_up, console_session.ses_name, len))
1830                 rc = -EFAULT;
1831
1832         LIBCFS_FREE(entp, sizeof(*entp));
1833
1834         return rc;
1835 }
1836
1837 int
1838 lstcon_session_end()
1839 {
1840         lstcon_rpc_trans_t *trans;
1841         lstcon_group_t     *grp;
1842         lstcon_batch_t     *bat;
1843         int                 rc = 0;
1844
1845         LASSERT (console_session.ses_state == LST_SESSION_ACTIVE);
1846
1847         rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list,
1848                                      NULL, LST_TRANS_SESEND, NULL,
1849                                      lstcon_sesrpc_condition, &trans);
1850         if (rc != 0) {
1851                 CERROR("Can't create transaction: %d\n", rc);
1852                 return rc;
1853         }
1854
1855         console_session.ses_shutdown = 1;
1856
1857         lstcon_rpc_pinger_stop();
1858
1859         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1860
1861         lstcon_rpc_trans_destroy(trans);
1862         /* User can do nothing even rpc failed, so go on */
1863
1864         /* waiting for orphan rpcs to die */
1865         lstcon_rpc_cleanup_wait();
1866
1867         console_session.ses_id    = LST_INVALID_SID;
1868         console_session.ses_state = LST_SESSION_NONE;
1869         console_session.ses_key   = 0;
1870         console_session.ses_force = 0;
1871         console_session.ses_feats_updated = 0;
1872
1873         /* destroy all batches */
1874         while (!list_empty(&console_session.ses_bat_list)) {
1875                 bat = list_entry(console_session.ses_bat_list.next,
1876                                  lstcon_batch_t, bat_link);
1877
1878                 lstcon_batch_destroy(bat);
1879         }
1880
1881         /* destroy all groups */
1882         while (!list_empty(&console_session.ses_grp_list)) {
1883                 grp = list_entry(console_session.ses_grp_list.next,
1884                                  lstcon_group_t, grp_link);
1885                 LASSERT(grp->grp_ref == 1);
1886
1887                 lstcon_group_put(grp);
1888         }
1889
1890         /* all nodes should be released */
1891         LASSERT(list_empty(&console_session.ses_ndl_list));
1892
1893         console_session.ses_shutdown = 0;
1894         console_session.ses_expired  = 0;
1895
1896         return rc;
1897 }
1898
1899 int
1900 lstcon_session_feats_check(unsigned feats)
1901 {
1902         int rc = 0;
1903
1904         if ((feats & ~LST_FEATS_MASK) != 0) {
1905                 CERROR("Can't support these features: %x\n",
1906                        (feats & ~LST_FEATS_MASK));
1907                 return -EPROTO;
1908         }
1909
1910         spin_lock(&console_session.ses_rpc_lock);
1911
1912         if (!console_session.ses_feats_updated) {
1913                 console_session.ses_feats_updated = 1;
1914                 console_session.ses_features = feats;
1915         }
1916
1917         if (console_session.ses_features != feats)
1918                 rc = -EPROTO;
1919
1920         spin_unlock(&console_session.ses_rpc_lock);
1921
1922         if (rc != 0) {
1923                 CERROR("remote features %x do not match with "
1924                        "session features %x of console\n",
1925                        feats, console_session.ses_features);
1926         }
1927
1928         return rc;
1929 }
1930
1931 static int
1932 lstcon_acceptor_handle (srpc_server_rpc_t *rpc)
1933 {
1934         srpc_msg_t        *rep  = &rpc->srpc_replymsg;
1935         srpc_msg_t        *req  = &rpc->srpc_reqstbuf->buf_msg;
1936         srpc_join_reqst_t *jreq = &req->msg_body.join_reqst;
1937         srpc_join_reply_t *jrep = &rep->msg_body.join_reply;
1938         lstcon_group_t    *grp  = NULL;
1939         lstcon_ndlink_t   *ndl;
1940         int                rc   = 0;
1941
1942         sfw_unpack_message(req);
1943
1944         mutex_lock(&console_session.ses_mutex);
1945
1946         jrep->join_sid = console_session.ses_id;
1947
1948         if (console_session.ses_id.ses_nid == LNET_NID_ANY) {
1949                 jrep->join_status = ESRCH;
1950                 goto out;
1951         }
1952
1953         if (lstcon_session_feats_check(req->msg_ses_feats) != 0) {
1954                 jrep->join_status = EPROTO;
1955                 goto out;
1956         }
1957
1958         if (jreq->join_sid.ses_nid != LNET_NID_ANY &&
1959              !lstcon_session_match(jreq->join_sid)) {
1960                 jrep->join_status = EBUSY;
1961                 goto out;
1962         }
1963
1964         if (lstcon_group_find(jreq->join_group, &grp) != 0) {
1965                 rc = lstcon_group_alloc(jreq->join_group, &grp);
1966                 if (rc != 0) {
1967                         CERROR("Out of memory\n");
1968                         goto out;
1969                 }
1970
1971                 list_add_tail(&grp->grp_link,
1972                               &console_session.ses_grp_list);
1973                 lstcon_group_addref(grp);
1974         }
1975
1976         if (grp->grp_ref > 2) {
1977                 /* Group in using */
1978                 jrep->join_status = EBUSY;
1979                 goto out;
1980         }
1981
1982         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0);
1983         if (rc == 0) {
1984                 jrep->join_status = EEXIST;
1985                 goto out;
1986         }
1987
1988         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1);
1989         if (rc != 0) {
1990                 CERROR("Out of memory\n");
1991                 goto out;
1992         }
1993
1994         ndl->ndl_node->nd_state   = LST_NODE_ACTIVE;
1995         ndl->ndl_node->nd_timeout = console_session.ses_timeout;
1996
1997         if (grp->grp_userland == 0)
1998                 grp->grp_userland = 1;
1999
2000         strlcpy(jrep->join_session, console_session.ses_name,
2001                 sizeof(jrep->join_session));
2002         jrep->join_timeout = console_session.ses_timeout;
2003         jrep->join_status  = 0;
2004
2005 out:
2006         rep->msg_ses_feats = console_session.ses_features;
2007         if (grp != NULL)
2008                 lstcon_group_put(grp);
2009
2010         mutex_unlock(&console_session.ses_mutex);
2011
2012         return rc;
2013 }
2014
2015 srpc_service_t lstcon_acceptor_service;
2016 void lstcon_init_acceptor_service(void)
2017 {
2018         /* initialize selftest console acceptor service table */
2019         lstcon_acceptor_service.sv_name    = "join session";
2020         lstcon_acceptor_service.sv_handler = lstcon_acceptor_handle;
2021         lstcon_acceptor_service.sv_id      = SRPC_SERVICE_JOIN;
2022         lstcon_acceptor_service.sv_wi_total = SFW_FRWK_WI_MAX;
2023 }
2024
2025 int lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_hdr *hdr);
2026
2027 DECLARE_IOCTL_HANDLER(lstcon_ioctl_handler, lstcon_ioctl_entry);
2028
2029 /* initialize console */
2030 int
2031 lstcon_console_init(void)
2032 {
2033         int     i;
2034         int     rc;
2035
2036         memset(&console_session, 0, sizeof(lstcon_session_t));
2037
2038         console_session.ses_id              = LST_INVALID_SID;
2039         console_session.ses_state           = LST_SESSION_NONE;
2040         console_session.ses_timeout         = 0;
2041         console_session.ses_force           = 0;
2042         console_session.ses_expired         = 0;
2043         console_session.ses_feats_updated   = 0;
2044         console_session.ses_features        = LST_FEATS_MASK;
2045         console_session.ses_laststamp       = cfs_time_current_sec();
2046
2047         mutex_init(&console_session.ses_mutex);
2048
2049         INIT_LIST_HEAD(&console_session.ses_ndl_list);
2050         INIT_LIST_HEAD(&console_session.ses_grp_list);
2051         INIT_LIST_HEAD(&console_session.ses_bat_list);
2052         INIT_LIST_HEAD(&console_session.ses_trans_list);
2053
2054         LIBCFS_ALLOC(console_session.ses_ndl_hash,
2055                      sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2056         if (console_session.ses_ndl_hash == NULL)
2057                 return -ENOMEM;
2058
2059         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
2060                 INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]);
2061
2062
2063         /* initialize acceptor service table */
2064         lstcon_init_acceptor_service();
2065
2066         rc = srpc_add_service(&lstcon_acceptor_service);
2067         LASSERT(rc != -EBUSY);
2068         if (rc != 0) {
2069                 LIBCFS_FREE(console_session.ses_ndl_hash,
2070                             sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2071                 return rc;
2072         }
2073
2074         rc = srpc_service_add_buffers(&lstcon_acceptor_service,
2075                                       lstcon_acceptor_service.sv_wi_total);
2076         if (rc != 0) {
2077                 rc = -ENOMEM;
2078                 goto out;
2079         }
2080
2081         rc = libcfs_register_ioctl(&lstcon_ioctl_handler);
2082
2083         if (rc == 0) {
2084                 lstcon_rpc_module_init();
2085                 return 0;
2086         }
2087
2088 out:
2089         srpc_shutdown_service(&lstcon_acceptor_service);
2090         srpc_remove_service(&lstcon_acceptor_service);
2091
2092         LIBCFS_FREE(console_session.ses_ndl_hash,
2093                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2094
2095         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2096
2097         return rc;
2098 }
2099
2100 int
2101 lstcon_console_fini(void)
2102 {
2103         int     i;
2104
2105         libcfs_deregister_ioctl(&lstcon_ioctl_handler);
2106
2107         mutex_lock(&console_session.ses_mutex);
2108
2109         srpc_shutdown_service(&lstcon_acceptor_service);
2110         srpc_remove_service(&lstcon_acceptor_service);
2111
2112         if (console_session.ses_state != LST_SESSION_NONE)
2113                 lstcon_session_end();
2114
2115         lstcon_rpc_module_fini();
2116
2117         mutex_unlock(&console_session.ses_mutex);
2118
2119         LASSERT(list_empty(&console_session.ses_ndl_list));
2120         LASSERT(list_empty(&console_session.ses_grp_list));
2121         LASSERT(list_empty(&console_session.ses_bat_list));
2122         LASSERT(list_empty(&console_session.ses_trans_list));
2123
2124         for (i = 0; i < LST_NODE_HASHSIZE; i++)
2125                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
2126
2127         LIBCFS_FREE(console_session.ses_ndl_hash,
2128                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2129
2130         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2131
2132         return 0;
2133 }
2134
2135 #endif