From: isaac Date: Fri, 22 Jun 2007 06:47:52 +0000 (+0000) Subject: - added LNET self test (landing b_self_test). X-Git-Tag: v1_7_100~43 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=776615e6825e2c90c2635c8b55e7bb02da33726c - added LNET self test (landing b_self_test). --- diff --git a/lnet/ChangeLog b/lnet/ChangeLog index 99aeb85..4fac91f 100644 --- a/lnet/ChangeLog +++ b/lnet/ChangeLog @@ -13,6 +13,11 @@ ptllnd - Portals 3.3 / UNICOS/lc 1.5.x, 2.0.x * bug fixes +Severity : major +Bugzilla : 10916 +Description: added LNET self test +Details : landing b_self_test + Severity : minor Frequency : rare Bugzilla : 12227 diff --git a/lnet/Makefile.in b/lnet/Makefile.in index 87b62f2..9109302 100644 --- a/lnet/Makefile.in +++ b/lnet/Makefile.in @@ -2,6 +2,7 @@ subdir-m += libcfs lnet-subdirs += lnet lnet-subdirs += klnds +lnet-subdirs += selftest subdir-m += $(lnet-subdirs) @INCLUDE_RULES@ diff --git a/lnet/autoMakefile.am b/lnet/autoMakefile.am index a0f7408..d8d062e 100644 --- a/lnet/autoMakefile.am +++ b/lnet/autoMakefile.am @@ -3,7 +3,7 @@ # This code is issued under the GNU General Public License. # See the file COPYING in this distribution -SUBDIRS = libcfs lnet klnds ulnds doc utils include \ +SUBDIRS = libcfs lnet klnds ulnds selftest doc utils include \ autoconf sources: diff --git a/lnet/autoconf/lustre-lnet.m4 b/lnet/autoconf/lustre-lnet.m4 index 30eda45..94d7475 100644 --- a/lnet/autoconf/lustre-lnet.m4 +++ b/lnet/autoconf/lustre-lnet.m4 @@ -1431,6 +1431,8 @@ lnet/libcfs/autoMakefile lnet/libcfs/linux/Makefile lnet/lnet/Makefile lnet/lnet/autoMakefile +lnet/selftest/Makefile +lnet/selftest/autoMakefile lnet/ulnds/Makefile lnet/ulnds/autoMakefile lnet/ulnds/socklnd/Makefile diff --git a/lnet/include/libcfs/kp30.h b/lnet/include/libcfs/kp30.h index 4ce9846..04aba75 100644 --- a/lnet/include/libcfs/kp30.h +++ b/lnet/include/libcfs/kp30.h @@ -545,6 +545,7 @@ extern int libcfs_ioctl_popdata(void *arg, void *buf, int size); #define IOC_LIBCFS_TESTPROTOCOMPAT _IOWR('e', 60, IOCTL_LIBCFS_TYPE) #define IOC_LIBCFS_PING _IOWR('e', 61, IOCTL_LIBCFS_TYPE) #define IOC_LIBCFS_DEBUG_PEER _IOWR('e', 62, IOCTL_LIBCFS_TYPE) +#define IOC_LIBCFS_LNETST _IOWR('e', 63, IOCTL_LIBCFS_TYPE) /* lnd ioctls */ #define IOC_LIBCFS_REGISTER_MYNID _IOWR('e', 70, IOCTL_LIBCFS_TYPE) #define IOC_LIBCFS_CLOSE_CONNECTION _IOWR('e', 71, IOCTL_LIBCFS_TYPE) diff --git a/lnet/include/lnet/Makefile.am b/lnet/include/lnet/Makefile.am index a6e5159..3328874 100644 --- a/lnet/include/lnet/Makefile.am +++ b/lnet/include/lnet/Makefile.am @@ -8,4 +8,4 @@ DIST_SUBDIRS := $(SUBDIRS) EXTRA_DIST = api.h api-support.h \ lib-lnet.h lib-types.h lnet.h lnetctl.h types.h \ - socklnd.h ptllnd.h ptllnd_wire.h + socklnd.h ptllnd.h ptllnd_wire.h lnetst.h diff --git a/lnet/include/lnet/lnetst.h b/lnet/include/lnet/lnetst.h new file mode 100644 index 0000000..768de14 --- /dev/null +++ b/lnet/include/lnet/lnetst.h @@ -0,0 +1,446 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Author: Liang Zhen + * + * This file is part of Lustre, http://www.lustre.org + */ + +#ifndef __LNET_ST_H__ +#define __LNET_ST_H__ + +#include +#include +#include + +#define LST_NAME_SIZE 32 /* max name buffer length */ + +#define LSTIO_DEBUG 0xC00 /* debug */ +#define LSTIO_SESSION_NEW 0xC01 /* create session */ +#define LSTIO_SESSION_END 0xC02 /* end session */ +#define LSTIO_SESSION_INFO 0xC03 /* query session */ +#define LSTIO_GROUP_ADD 0xC10 /* add group */ +#define LSTIO_GROUP_LIST 0xC11 /* list all groups in session */ +#define LSTIO_GROUP_INFO 0xC12 /* query defailt infomation of specified group */ +#define LSTIO_GROUP_DEL 0xC13 /* delete group */ +#define LSTIO_NODES_ADD 0xC14 /* add nodes to specified group */ +#define LSTIO_GROUP_UPDATE 0xC15 /* update group */ +#define LSTIO_BATCH_ADD 0xC20 /* add batch */ +#define LSTIO_BATCH_START 0xC21 /* start batch */ +#define LSTIO_BATCH_STOP 0xC22 /* stop batch */ +#define LSTIO_BATCH_DEL 0xC23 /* delete batch */ +#define LSTIO_BATCH_LIST 0xC24 /* show all batches in the session */ +#define LSTIO_BATCH_INFO 0xC25 /* show defail of specified batch */ +#define LSTIO_TEST_ADD 0xC26 /* add test (to batch) */ +#define LSTIO_BATCH_QUERY 0xC27 /* query batch status */ +#define LSTIO_STAT_QUERY 0xC30 /* get stats */ + +typedef struct { + lnet_nid_t ses_nid; /* nid of console node */ + __u64 ses_stamp; /* time stamp */ +} lst_sid_t; /*** session id */ + +#define LST_INVALID_SID ((const lst_sid_t){.ses_nid = LNET_NID_ANY,\ + .ses_stamp = -1}) + +typedef struct { + __u64 bat_id; /* unique id in session */ +} lst_bid_t; /*** batch id (group of tests) */ + +/* Status of test node */ +#define LST_NODE_ACTIVE 0x1 /* node in this session */ +#define LST_NODE_BUSY 0x2 /* node is taken by other session */ +#define LST_NODE_DOWN 0x4 /* node is down */ +#define LST_NODE_UNKNOWN 0x8 /* node not in session */ + +typedef struct { + lnet_process_id_t nde_id; /* id of node */ + int nde_state; /* state of node */ +} lstcon_node_ent_t; /*** node entry, for list_group command */ + +typedef struct { + int nle_nnode; /* # of nodes */ + int nle_nactive; /* # of active nodes */ + int nle_nbusy; /* # of busy nodes */ + int nle_ndown; /* # of down nodes */ + int nle_nunknown; /* # of unknown nodes */ +} lstcon_ndlist_ent_t; /*** node_list entry, for list_batch command */ + +typedef struct { + int tse_type; /* test type */ + int tse_loop; /* loop count */ + int tse_concur; /* concurrency of test */ +} lstcon_test_ent_t; /*** test summary entry, for list_batch command */ + +typedef struct { + int bae_state; /* batch status */ + int bae_timeout; /* batch timeout */ + int bae_ntest; /* # of tests in the batch */ +} lstcon_batch_ent_t; /*** batch summary entry, for list_batch command */ + +typedef struct { + lstcon_ndlist_ent_t tbe_cli_nle; /* client (group) node_list entry */ + lstcon_ndlist_ent_t tbe_srv_nle; /* server (group) node_list entry */ + union { + lstcon_test_ent_t tbe_test; /* test entry */ + lstcon_batch_ent_t tbe_batch; /* batch entry */ + } u; +} lstcon_test_batch_ent_t; /*** test/batch verbose information entry, + *** for list_batch command */ + +typedef struct { + struct list_head rpe_link; /* link chain */ + lnet_process_id_t rpe_peer; /* peer's id */ + struct timeval rpe_stamp; /* time stamp of RPC */ + int rpe_state; /* peer's state */ + int rpe_rpc_errno; /* RPC errno */ + + lst_sid_t rpe_sid; /* peer's session id */ + int rpe_fwk_errno; /* framework errno */ + int rpe_priv[4]; /* private data */ + char rpe_payload[0]; /* private reply payload */ +} lstcon_rpc_ent_t; + +typedef struct { + int trs_rpc_stat[4]; /* RPCs stat (0: total, 1: failed, 2: finished, 4: reserved */ + int trs_rpc_errno; /* RPC errno */ + int trs_fwk_stat[8]; /* framework stat */ + int trs_fwk_errno; /* errno of the first remote error */ + void *trs_fwk_private; /* private framework stat */ +} lstcon_trans_stat_t; + +static inline int +lstcon_rpc_stat_total(lstcon_trans_stat_t *stat, int inc) +{ + return inc ? ++stat->trs_rpc_stat[0] : stat->trs_rpc_stat[0]; +} + +static inline int +lstcon_rpc_stat_success(lstcon_trans_stat_t *stat, int inc) +{ + return inc ? ++stat->trs_rpc_stat[1] : stat->trs_rpc_stat[1]; +} + +static inline int +lstcon_rpc_stat_failure(lstcon_trans_stat_t *stat, int inc) +{ + return inc ? ++stat->trs_rpc_stat[2] : stat->trs_rpc_stat[2]; +} + +static inline int +lstcon_sesop_stat_success(lstcon_trans_stat_t *stat, int inc) +{ + return inc ? ++stat->trs_fwk_stat[0] : stat->trs_fwk_stat[0]; +} + +static inline int +lstcon_sesop_stat_failure(lstcon_trans_stat_t *stat, int inc) +{ + return inc ? ++stat->trs_fwk_stat[1] : stat->trs_fwk_stat[1]; +} + +static inline int +lstcon_sesqry_stat_active(lstcon_trans_stat_t *stat, int inc) +{ + return inc ? ++stat->trs_fwk_stat[0] : stat->trs_fwk_stat[0]; +} + +static inline int +lstcon_sesqry_stat_busy(lstcon_trans_stat_t *stat, int inc) +{ + return inc ? ++stat->trs_fwk_stat[1] : stat->trs_fwk_stat[1]; +} + +static inline int +lstcon_sesqry_stat_unknown(lstcon_trans_stat_t *stat, int inc) +{ + return inc ? ++stat->trs_fwk_stat[2] : stat->trs_fwk_stat[2]; +} + +static inline int +lstcon_tsbop_stat_success(lstcon_trans_stat_t *stat, int inc) +{ + return inc ? ++stat->trs_fwk_stat[0] : stat->trs_fwk_stat[0]; +} + +static inline int +lstcon_tsbop_stat_failure(lstcon_trans_stat_t *stat, int inc) +{ + return inc ? ++stat->trs_fwk_stat[1] : stat->trs_fwk_stat[1]; +} + +static inline int +lstcon_tsbqry_stat_idle(lstcon_trans_stat_t *stat, int inc) +{ + return inc ? ++stat->trs_fwk_stat[0] : stat->trs_fwk_stat[0]; +} + +static inline int +lstcon_tsbqry_stat_run(lstcon_trans_stat_t *stat, int inc) +{ + return inc ? ++stat->trs_fwk_stat[1] : stat->trs_fwk_stat[1]; +} + +static inline int +lstcon_tsbqry_stat_failure(lstcon_trans_stat_t *stat, int inc) +{ + return inc ? ++stat->trs_fwk_stat[2] : stat->trs_fwk_stat[2]; +} + +static inline int +lstcon_statqry_stat_success(lstcon_trans_stat_t *stat, int inc) +{ + return inc ? ++stat->trs_fwk_stat[0] : stat->trs_fwk_stat[0]; +} + +static inline int +lstcon_statqry_stat_failure(lstcon_trans_stat_t *stat, int inc) +{ + return inc ? ++stat->trs_fwk_stat[1] : stat->trs_fwk_stat[1]; +} + +/* create a session */ +typedef struct { + int lstio_ses_key; /* IN: local key */ + int lstio_ses_timeout; /* IN: session timeout */ + int lstio_ses_force; /* IN: force create ? */ + lst_sid_t *lstio_ses_idp; /* OUT: session id */ + int lstio_ses_nmlen; /* IN: name length */ + char *lstio_ses_namep; /* IN: session name */ +} lstio_session_new_args_t; + +/* query current session */ +typedef struct { + lst_sid_t *lstio_ses_idp; /* OUT: session id */ + int *lstio_ses_keyp; /* OUT: local key */ + lstcon_ndlist_ent_t *lstio_ses_ndinfo; /* OUT: */ + int lstio_ses_nmlen; /* IN: name length */ + char *lstio_ses_namep; /* OUT: session name */ +} lstio_session_info_args_t; + +/* delete a session */ +typedef struct { + int lstio_ses_key; /* IN: session key */ +} lstio_session_end_args_t; + +#define LST_OPC_SESSION 1 +#define LST_OPC_GROUP 2 +#define LST_OPC_NODES 3 +#define LST_OPC_BATCHCLI 4 +#define LST_OPC_BATCHSRV 5 + +typedef struct { + int lstio_dbg_key; /* IN: session key */ + int lstio_dbg_type; /* IN: debug sessin|batch|group|nodes list */ + int lstio_dbg_flags; /* IN: reserved debug flags */ + int lstio_dbg_timeout; /* IN: timeout of debug */ + + int lstio_dbg_nmlen; /* IN: len of name */ + char *lstio_dbg_namep; /* IN: name of group|batch */ + int lstio_dbg_count; /* IN: # of test nodes to debug */ + lnet_process_id_t *lstio_dbg_idsp; /* IN: id of test nodes */ + struct list_head *lstio_dbg_resultp; /* OUT: list head of result buffer */ +} lstio_debug_args_t; + +typedef struct { + int lstio_grp_key; /* IN: session key */ + int lstio_grp_nmlen; /* IN: name length */ + char *lstio_grp_namep; /* IN: group name */ +} lstio_group_add_args_t; + +typedef struct { + int lstio_grp_key; /* IN: session key */ + int lstio_grp_nmlen; /* IN: name length */ + char *lstio_grp_namep; /* IN: group name */ +} lstio_group_del_args_t; + +#define LST_GROUP_CLEAN 1 /* remove inactive nodes in the group */ +#define LST_GROUP_REFRESH 2 /* refresh inactive nodes in the group */ +#define LST_GROUP_RMND 3 /* delete nodes from the group */ + +typedef struct { + int lstio_grp_key; /* IN: session key */ + int lstio_grp_opc; /* IN: OPC */ + int lstio_grp_args; /* IN: arguments */ + int lstio_grp_nmlen; /* IN: name length */ + char *lstio_grp_namep; /* IN: group name */ + int lstio_grp_count; /* IN: # of nodes id */ + lnet_process_id_t *lstio_grp_idsp; /* IN: array of nodes */ + struct list_head *lstio_grp_resultp; /* OUT: list head of result buffer */ +} lstio_group_update_args_t; + +typedef struct { + int lstio_grp_key; /* IN: session key */ + int lstio_grp_nmlen; /* IN: name length */ + char *lstio_grp_namep; /* IN: group name */ + int lstio_grp_count; /* IN: # of nodes */ + lnet_process_id_t *lstio_grp_idsp; /* IN: nodes */ + struct list_head *lstio_grp_resultp; /* OUT: list head of result buffer */ +} lstio_group_nodes_args_t; + +typedef struct { + int lstio_grp_key; /* IN: session key */ + int lstio_grp_idx; /* IN: group idx */ + int lstio_grp_nmlen; /* IN: name len */ + char *lstio_grp_namep; /* OUT: name */ +} lstio_group_list_args_t; + +typedef struct { + int lstio_grp_key; /* IN: session key */ + int lstio_grp_nmlen; /* IN: name len */ + char *lstio_grp_namep; /* IN: name */ + lstcon_ndlist_ent_t *lstio_grp_entp; /* OUT: description of group */ + + int *lstio_grp_idxp; /* IN/OUT: node index */ + int *lstio_grp_ndentp; /* IN/OUT: # of nodent */ + lstcon_node_ent_t *lstio_grp_dentsp; /* OUT: nodent array */ +} lstio_group_info_args_t; + +#define LST_DEFAULT_BATCH "batch" /* default batch name */ + +typedef struct { + int lstio_bat_key; /* IN: session key */ + int lstio_bat_nmlen; /* IN: name length */ + char *lstio_bat_namep; /* IN: batch name */ +} lstio_batch_add_args_t; + +typedef struct { + int lstio_bat_key; /* IN: session key */ + int lstio_bat_nmlen; /* IN: name length */ + char *lstio_bat_namep; /* IN: batch name */ +} lstio_batch_del_args_t; + +typedef struct { + int lstio_bat_key; /* IN: session key */ + int lstio_bat_timeout; /* IN: timeout for the batch */ + int lstio_bat_nmlen; /* IN: name length */ + char *lstio_bat_namep; /* IN: batch name */ + struct list_head *lstio_bat_resultp; /* OUT: list head of result buffer */ +} lstio_batch_run_args_t; + +typedef struct { + int lstio_bat_key; /* IN: session key */ + int lstio_bat_force; /* IN: abort unfinished test RPC */ + int lstio_bat_nmlen; /* IN: name length */ + char *lstio_bat_namep; /* IN: batch name */ + struct list_head *lstio_bat_resultp; /* OUT: list head of result buffer */ +} lstio_batch_stop_args_t; + +typedef struct { + int lstio_bat_key; /* IN: session key */ + int lstio_bat_testidx; /* IN: test index */ + int lstio_bat_client; /* IN: is test client? */ + int lstio_bat_timeout; /* IN: timeout for waiting */ + int lstio_bat_nmlen; /* IN: name length */ + char *lstio_bat_namep; /* IN: batch name */ + struct list_head *lstio_bat_resultp; /* OUT: list head of result buffer */ +} lstio_batch_query_args_t; + +typedef struct { + int lstio_bat_key; /* IN: session key */ + int lstio_bat_idx; /* IN: index */ + int lstio_bat_nmlen; /* IN: name length */ + char *lstio_bat_namep; /* IN: batch name */ +} lstio_batch_list_args_t; + +typedef struct { + int lstio_bat_key; /* IN: session key */ + int lstio_bat_nmlen; /* IN: name length */ + char *lstio_bat_namep; /* IN: name */ + int lstio_bat_server; /* IN: query server or not */ + int lstio_bat_testidx; /* IN: test index */ + lstcon_test_batch_ent_t *lstio_bat_entp; /* OUT: batch ent */ + + int *lstio_bat_idxp; /* IN/OUT: index of node */ + int *lstio_bat_ndentp; /* IN/OUT: # of nodent */ + lstcon_node_ent_t *lstio_bat_dentsp; /* array of nodent */ +} lstio_batch_info_args_t; + +/* add stat in session */ +typedef struct { + int lstio_sta_key; /* IN: session key */ + int lstio_sta_timeout; /* IN: timeout for stat requst */ + int lstio_sta_nmlen; /* IN: group name length */ + char *lstio_sta_namep; /* IN: group name */ + int lstio_sta_count; /* IN: # of pid */ + lnet_process_id_t *lstio_sta_idsp; /* IN: pid */ + struct list_head *lstio_sta_resultp; /* OUT: list head of result buffer */ +} lstio_stat_args_t; + +typedef enum { + LST_TEST_BULK = 1, + LST_TEST_PING = 2 +} lst_test_type_t; + +/* create a test in a batch */ +#define LST_MAX_CONCUR 1024 /* Max concurrency of test */ + +typedef struct { + int lstio_tes_key; /* IN: session key */ + int lstio_tes_bat_nmlen; /* IN: batch name len */ + char *lstio_tes_bat_name; /* IN: batch name */ + int lstio_tes_type; /* IN: test type */ + int lstio_tes_oneside; /* IN: one sided test */ + int lstio_tes_loop; /* IN: loop count */ + int lstio_tes_concur; /* IN: concurrency */ + + int lstio_tes_dist; /* IN: node distribution in destination groups */ + int lstio_tes_span; /* IN: node span in destination groups */ + int lstio_tes_sgrp_nmlen; /* IN: source group name length */ + char *lstio_tes_sgrp_name; /* IN: group name */ + int lstio_tes_dgrp_nmlen; /* IN: destination group name length */ + char *lstio_tes_dgrp_name; /* IN: group name */ + + int lstio_tes_param_len; /* IN: param buffer len */ + void *lstio_tes_param; /* IN: parameter for specified test: + lstio_bulk_param_t, + lstio_ping_param_t, + ... more */ + struct list_head *lstio_tes_resultp; /* OUT: list head of result buffer */ +} lstio_test_args_t; + +typedef enum { + LST_BRW_READ = 1, + LST_BRW_WRITE = 2 +} lst_brw_type_t; + +typedef enum { + LST_BRW_CHECK_NONE = 1, + LST_BRW_CHECK_SIMPLE = 2, + LST_BRW_CHECK_FULL = 3 +} lst_brw_flags_t; + +typedef struct { + int blk_opc; /* bulk operation code */ + int blk_npg; /* # pages */ + int blk_time; /* time of running the test*/ + int blk_flags; /* reserved flags */ +} lst_test_bulk_param_t, lst_bulk_param_t; + +typedef struct { + int png_size; /* size of ping message */ + int png_time; /* time */ + int png_loop; /* loop */ + int png_flags; /* reserved flags */ +} lst_test_ping_param_t, lst_ping_param_t; + +/* more tests */ + +typedef struct { + __u32 errors; + __u32 rpcs_sent; + __u32 rpcs_rcvd; + __u32 rpcs_dropped; + __u32 rpcs_expired; + __u64 bulk_get; + __u64 bulk_put; +} srpc_counters_t; + +typedef struct { + __u32 active_tests; + __u32 active_batches; + __u32 zombie_sessions; + __u32 brw_errors; +} sfw_counters_t; + +#endif diff --git a/lnet/lnet/lib-md.c b/lnet/lnet/lib-md.c index 0e8524c..57d92ae 100644 --- a/lnet/lnet/lib-md.c +++ b/lnet/lnet/lib-md.c @@ -44,7 +44,7 @@ lnet_md_unlink(lnet_libmd_t *md) lnet_me_unlink(me); } - /* emsure all future handle lookups fail */ + /* ensure all future handle lookups fail */ lnet_invalidate_handle(&md->md_lh); } diff --git a/lnet/selftest/.cvsignore b/lnet/selftest/.cvsignore new file mode 100644 index 0000000..5ed596b --- /dev/null +++ b/lnet/selftest/.cvsignore @@ -0,0 +1,10 @@ +.deps +Makefile +.*.cmd +autoMakefile.in +autoMakefile +*.ko +*.mod.c +.*.flags +.tmp_versions +.depend diff --git a/lnet/selftest/Makefile.in b/lnet/selftest/Makefile.in new file mode 100644 index 0000000..8ebef75 --- /dev/null +++ b/lnet/selftest/Makefile.in @@ -0,0 +1,7 @@ +MODULES := lnet_selftest + +lnet_selftest-objs := console.o conrpc.o conctl.o framework.o timer.o rpc.o workitem.o module.o ping_test.o brw_test.o + +default: all + +@INCLUDE_RULES@ diff --git a/lnet/selftest/autoMakefile.am b/lnet/selftest/autoMakefile.am new file mode 100644 index 0000000..36af901 --- /dev/null +++ b/lnet/selftest/autoMakefile.am @@ -0,0 +1,23 @@ +my_sources = console.c conrpc.c conctl.c console.h conrpc.h \ + framework.c timer.c rpc.c workitem.c module.c \ + ping_test.c brw_test.c + +if LIBLUSTRE +noinst_LIBRARIES= libselftest.a +libselftest_a_SOURCES= $(my_sources) +libselftest_a_CPPFLAGS = $(LLCPPFLAGS) +libselftest_a_CFLAGS = $(LLCFLAGS) +endif + +if MODULES + +if LINUX +modulenet_DATA = lnet_selftest$(KMODEXT) +endif # LINUX + +endif # MODULES + +install-data-hook: $(install_data_hook) + +MOSTLYCLEANFILES = @MOSTLYCLEANFILES@ selftest +DIST_SOURCES = $(lnet_selftest-objs:%.o=%.c) console.h conrpc.h rpc.h selftest.h timer.h diff --git a/lnet/selftest/brw_test.c b/lnet/selftest/brw_test.c new file mode 100644 index 0000000..ee2d024 --- /dev/null +++ b/lnet/selftest/brw_test.c @@ -0,0 +1,397 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2001, 2002 Cluster File Systems, Inc. + * Author: Isaac Huang + * + */ + +#include +#include "selftest.h" + + +static int brw_inject_errors = 0; +CFS_MODULE_PARM(brw_inject_errors, "i", int, 0644, + "# data errors to inject randomly"); + +static void +brw_client_fini (sfw_test_instance_t *tsi) +{ + srpc_bulk_t *bulk; + sfw_test_unit_t *tsu; + + list_for_each_entry (tsu, &tsi->tsi_units, tsu_list) { + bulk = tsu->tsu_private; + if (bulk == NULL) continue; + + srpc_free_bulk(bulk); + tsu->tsu_private = NULL; + } +} + +int +brw_client_init (sfw_test_instance_t *tsi) +{ + test_bulk_req_t *breq = &tsi->tsi_u.bulk; + int flags = breq->blk_flags; + int npg = breq->blk_npg; + srpc_bulk_t *bulk; + sfw_test_unit_t *tsu; + + if (npg > LNET_MAX_IOV || npg <= 0) + return -EINVAL; + + if (breq->blk_opc != LST_BRW_READ && breq->blk_opc != LST_BRW_WRITE) + return -EINVAL; + + if (flags != LST_BRW_CHECK_NONE && + flags != LST_BRW_CHECK_FULL && flags != LST_BRW_CHECK_SIMPLE) + return -EINVAL; + + list_for_each_entry (tsu, &tsi->tsi_units, tsu_list) { + bulk = srpc_alloc_bulk(npg, breq->blk_opc == LST_BRW_READ); + if (bulk == NULL) { + brw_client_fini(tsi); + return -ENOMEM; + } + + tsu->tsu_private = bulk; + } + + return 0; +} + +#define BRW_POISON 0xbeefbeefbeefbeefULL +#define BRW_MAGIC 0xeeb0eeb1eeb2eeb3ULL +#define BRW_MSIZE sizeof(__u64) + +int +brw_inject_one_error (void) +{ + struct timeval tv; + + if (brw_inject_errors <= 0) return 0; + +#ifndef __KERNEL__ + gettimeofday(&tv, NULL); +#else + do_gettimeofday(&tv); +#endif + + if ((tv.tv_usec & 1) == 0) return 0; + + return brw_inject_errors--; +} + +void +brw_fill_page (cfs_page_t *pg, int pattern, __u64 magic) +{ + char *addr = cfs_page_address(pg); + int i; + + LASSERT (addr != NULL); + + if (pattern == LST_BRW_CHECK_NONE) + return; + + if (magic == BRW_MAGIC) + magic += brw_inject_one_error(); + + if (pattern == LST_BRW_CHECK_SIMPLE) { + memcpy(addr, &magic, BRW_MSIZE); + addr += CFS_PAGE_SIZE - BRW_MSIZE; + memcpy(addr, &magic, BRW_MSIZE); + return; + } + + if (pattern == LST_BRW_CHECK_FULL) { + for (i = 0; i < CFS_PAGE_SIZE / BRW_MSIZE; i++) + memcpy(addr + i * BRW_MSIZE, &magic, BRW_MSIZE); + return; + } + + LBUG (); + return; +} + +int +brw_check_page (cfs_page_t *pg, int pattern, __u64 magic) +{ + char *addr = cfs_page_address(pg); + __u64 data; + int i; + + LASSERT (addr != NULL); + + if (pattern == LST_BRW_CHECK_NONE) + return 0; + + if (pattern == LST_BRW_CHECK_SIMPLE) { + data = *((__u64 *) addr); + if (data != magic) goto bad_data; + + addr += CFS_PAGE_SIZE - BRW_MSIZE; + data = *((__u64 *) addr); + if (data != magic) goto bad_data; + + return 0; + } + + if (pattern == LST_BRW_CHECK_FULL) { + for (i = 0; i < CFS_PAGE_SIZE / BRW_MSIZE; i++) { + data = *(((__u64 *) addr) + i); + if (data != magic) goto bad_data; + } + + return 0; + } + + LBUG (); + +bad_data: + CERROR ("Bad data in page %p: "LPU64", "LPU64" expected\n", + pg, data, magic); + return 1; +} + +void +brw_fill_bulk (srpc_bulk_t *bk, int pattern, __u64 magic) +{ + int i; + cfs_page_t *pg; + + for (i = 0; i < bk->bk_niov; i++) { +#ifdef __KERNEL__ + pg = bk->bk_iovs[i].kiov_page; +#else + pg = bk->bk_pages[i]; +#endif + brw_fill_page(pg, pattern, magic); + } +} + +int +brw_check_bulk (srpc_bulk_t *bk, int pattern, __u64 magic) +{ + int i; + cfs_page_t *pg; + + for (i = 0; i < bk->bk_niov; i++) { +#ifdef __KERNEL__ + pg = bk->bk_iovs[i].kiov_page; +#else + pg = bk->bk_pages[i]; +#endif + if (brw_check_page(pg, pattern, magic) != 0) { + CERROR ("Bulk page %p (%d/%d) is corrupted!\n", + pg, i, bk->bk_niov); + return 1; + } + } + + return 0; +} + +static int +brw_client_prep_rpc (sfw_test_unit_t *tsu, + lnet_process_id_t dest, srpc_client_rpc_t **rpcpp) +{ + srpc_bulk_t *bulk = tsu->tsu_private; + sfw_test_instance_t *tsi = tsu->tsu_instance; + test_bulk_req_t *breq = &tsi->tsi_u.bulk; + int npg = breq->blk_npg; + int flags = breq->blk_flags; + srpc_client_rpc_t *rpc; + srpc_brw_reqst_t *req; + int rc; + + LASSERT (bulk != NULL); + LASSERT (bulk->bk_niov == npg); + + rc = sfw_create_test_rpc(tsu, dest, npg, npg * CFS_PAGE_SIZE, &rpc); + if (rc != 0) return rc; + + memcpy(&rpc->crpc_bulk, bulk, offsetof(srpc_bulk_t, bk_iovs[npg])); + if (breq->blk_opc == LST_BRW_WRITE) + brw_fill_bulk(&rpc->crpc_bulk, flags, BRW_MAGIC); + else + brw_fill_bulk(&rpc->crpc_bulk, flags, BRW_POISON); + + req = &rpc->crpc_reqstmsg.msg_body.brw_reqst; + req->brw_flags = flags; + req->brw_rw = breq->blk_opc; + req->brw_len = npg * CFS_PAGE_SIZE; + + *rpcpp = rpc; + return 0; +} + +static void +brw_client_done_rpc (sfw_test_unit_t *tsu, srpc_client_rpc_t *rpc) +{ + __u64 magic = BRW_MAGIC; + srpc_msg_t *msg = &rpc->crpc_replymsg; + srpc_brw_reply_t *reply = &msg->msg_body.brw_reply; + srpc_brw_reqst_t *reqst = &rpc->crpc_reqstmsg.msg_body.brw_reqst; + sfw_session_t *sn = tsu->tsu_instance->tsi_batch->bat_session; + + LASSERT (sn != NULL); + +#ifndef __KERNEL__ + rpc->crpc_bulk.bk_pages = NULL; +#endif + + if (rpc->crpc_status != 0) { + CERROR ("BRW RPC to %s failed with %d\n", + libcfs_id2str(rpc->crpc_dest), rpc->crpc_status); + return; + } + + if (msg->msg_magic != SRPC_MSG_MAGIC) { + __swab64s(&magic); + __swab32s(&reply->brw_status); + } + + if (tsu->tsu_error == 0) + tsu->tsu_error = -reply->brw_status; + + CDEBUG (reply->brw_status ? D_WARNING : D_NET, + "BRW RPC to %s finished with brw_status: %d\n", + libcfs_id2str(rpc->crpc_dest), reply->brw_status); + + if (reply->brw_status != 0) { + atomic_inc(&sn->sn_brw_errors); + return; + } + + if (reqst->brw_rw == LST_BRW_WRITE) return; + + if (brw_check_bulk(&rpc->crpc_bulk, reqst->brw_flags, magic) != 0) { + CERROR ("Bulk data from %s is corrupted!\n", + libcfs_id2str(rpc->crpc_dest)); + tsu->tsu_error = -EBADMSG; + atomic_inc(&sn->sn_brw_errors); + } + + return; +} + +void +brw_server_rpc_done (srpc_server_rpc_t *rpc) +{ + srpc_bulk_t *blk = rpc->srpc_bulk; + + if (blk == NULL) return; + + if (rpc->srpc_status != 0) + CERROR ("Bulk transfer %s %s has failed: %d\n", + blk->bk_sink ? "from" : "to", + libcfs_id2str(rpc->srpc_peer), rpc->srpc_status); + else + CDEBUG (D_NET, "Transfered %d pages bulk data %s %s\n", + blk->bk_niov, blk->bk_sink ? "from" : "to", + libcfs_id2str(rpc->srpc_peer)); + + sfw_free_pages(rpc); +} + +int +brw_bulk_ready (srpc_server_rpc_t *rpc, int status) +{ + __u64 magic = BRW_MAGIC; + srpc_brw_reply_t *reply = &rpc->srpc_replymsg.msg_body.brw_reply; + srpc_brw_reqst_t *reqst; + srpc_msg_t *reqstmsg; + + LASSERT (rpc->srpc_bulk != NULL); + LASSERT (rpc->srpc_reqstbuf != NULL); + + reqstmsg = &rpc->srpc_reqstbuf->buf_msg; + reqst = &reqstmsg->msg_body.brw_reqst; + + if (status != 0) { + CERROR ("BRW bulk %s failed for RPC from %s: %d\n", + reqst->brw_rw == LST_BRW_READ ? "READ" : "WRITE", + libcfs_id2str(rpc->srpc_peer), status); + return -EIO; + } + + if (reqst->brw_rw == LST_BRW_READ) + return 0; + + if (reqstmsg->msg_magic != SRPC_MSG_MAGIC) + __swab64s(&magic); + + if (brw_check_bulk(rpc->srpc_bulk, reqst->brw_flags, magic) != 0) { + CERROR ("Bulk data from %s is corrupted!\n", + libcfs_id2str(rpc->srpc_peer)); + reply->brw_status = EBADMSG; + } + + return 0; +} + +int +brw_server_handle (srpc_server_rpc_t *rpc) +{ + srpc_service_t *sv = rpc->srpc_service; + srpc_msg_t *replymsg = &rpc->srpc_replymsg; + srpc_msg_t *reqstmsg = &rpc->srpc_reqstbuf->buf_msg; + srpc_brw_reply_t *reply = &replymsg->msg_body.brw_reply; + srpc_brw_reqst_t *reqst = &reqstmsg->msg_body.brw_reqst; + int rc; + + LASSERT (sv->sv_id == SRPC_SERVICE_BRW); + + if (reqstmsg->msg_magic != SRPC_MSG_MAGIC) { + LASSERT (reqstmsg->msg_magic == __swab32(SRPC_MSG_MAGIC)); + + __swab32s(&reqstmsg->msg_type); + __swab32s(&reqst->brw_rw); + __swab32s(&reqst->brw_len); + __swab32s(&reqst->brw_flags); + __swab64s(&reqst->brw_rpyid); + __swab64s(&reqst->brw_bulkid); + } + LASSERT (reqstmsg->msg_type == srpc_service2request(sv->sv_id)); + + rpc->srpc_done = brw_server_rpc_done; + + if ((reqst->brw_rw != LST_BRW_READ && reqst->brw_rw != LST_BRW_WRITE) || + reqst->brw_len == 0 || (reqst->brw_len & ~CFS_PAGE_MASK) != 0 || + reqst->brw_len / CFS_PAGE_SIZE > LNET_MAX_IOV || + (reqst->brw_flags != LST_BRW_CHECK_NONE && + reqst->brw_flags != LST_BRW_CHECK_FULL && + reqst->brw_flags != LST_BRW_CHECK_SIMPLE)) { + reply->brw_status = EINVAL; + return 0; + } + + reply->brw_status = 0; + rc = sfw_alloc_pages(rpc, reqst->brw_len / CFS_PAGE_SIZE, + reqst->brw_rw == LST_BRW_WRITE); + if (rc != 0) return rc; + + if (reqst->brw_rw == LST_BRW_READ) + brw_fill_bulk(rpc->srpc_bulk, reqst->brw_flags, BRW_MAGIC); + else + brw_fill_bulk(rpc->srpc_bulk, reqst->brw_flags, BRW_POISON); + + return 0; +} + +sfw_test_client_ops_t brw_test_client = +{ + .tso_init = brw_client_init, + .tso_fini = brw_client_fini, + .tso_prep_rpc = brw_client_prep_rpc, + .tso_done_rpc = brw_client_done_rpc, +}; + +srpc_service_t brw_test_service = +{ + .sv_name = "brw test", + .sv_handler = brw_server_handle, + .sv_bulk_ready = brw_bulk_ready, + .sv_id = SRPC_SERVICE_BRW, +}; diff --git a/lnet/selftest/conctl.c b/lnet/selftest/conctl.c new file mode 100644 index 0000000..7cb1378 --- /dev/null +++ b/lnet/selftest/conctl.c @@ -0,0 +1,879 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Author: Liang Zhen + * + * This file is part of Lustre, http://www.lustre.org + * + * IOC handle in kernel + */ +#ifdef __KERNEL__ + +#include +#include +#include +#include "console.h" + +int +lst_session_new_ioctl(lstio_session_new_args_t *args) +{ + char *name; + int rc; + + if (args->lstio_ses_idp == NULL || /* address for output sid */ + args->lstio_ses_key == 0 || /* no key is specified */ + args->lstio_ses_namep == NULL || /* session name */ + args->lstio_ses_nmlen <= 0 || + args->lstio_ses_nmlen > LST_NAME_SIZE) + return -EINVAL; + + LIBCFS_ALLOC(name, args->lstio_ses_nmlen + 1); + if (name == NULL) + return -ENOMEM; + + if (copy_from_user(name, + args->lstio_ses_namep, + args->lstio_ses_nmlen)) { + LIBCFS_FREE(name, args->lstio_ses_nmlen + 1); + return -EFAULT; + } + + name[args->lstio_ses_nmlen] = 0; + + rc = lstcon_session_new(name, + args->lstio_ses_key, + args->lstio_ses_timeout, + args->lstio_ses_force, + args->lstio_ses_idp); + + LIBCFS_FREE(name, args->lstio_ses_nmlen + 1); + + return rc; +} + +int +lst_session_end_ioctl(lstio_session_end_args_t *args) +{ + if (args->lstio_ses_key != console_session.ses_key) + return -EACCES; + + return lstcon_session_end(); +} + +int +lst_session_info_ioctl(lstio_session_info_args_t *args) +{ + /* no checking of key */ + + if (args->lstio_ses_idp == NULL || /* address for ouput sid */ + args->lstio_ses_keyp == NULL || /* address for ouput key */ + args->lstio_ses_ndinfo == NULL || /* address for output ndinfo */ + args->lstio_ses_namep == NULL || /* address for ouput name */ + args->lstio_ses_nmlen <= 0 || + args->lstio_ses_nmlen > LST_NAME_SIZE) + return -EINVAL; + + return lstcon_session_info(args->lstio_ses_idp, + args->lstio_ses_keyp, + args->lstio_ses_ndinfo, + args->lstio_ses_namep, + args->lstio_ses_nmlen); +} + +int +lst_debug_ioctl(lstio_debug_args_t *args) +{ + char *name = NULL; + int client = 1; + int rc; + + if (args->lstio_dbg_key != console_session.ses_key) + return -EACCES; + + if (args->lstio_dbg_resultp == NULL) + return -EINVAL; + + if (args->lstio_dbg_namep != NULL && /* name of batch/group */ + (args->lstio_dbg_nmlen <= 0 || + args->lstio_dbg_nmlen > LST_NAME_SIZE)) + return -EINVAL; + + if (args->lstio_dbg_namep != NULL) { + LIBCFS_ALLOC(name, args->lstio_dbg_nmlen + 1); + if (name == NULL) + return -ENOMEM; + + if (copy_from_user(name, args->lstio_dbg_namep, + args->lstio_dbg_nmlen)) { + LIBCFS_FREE(name, args->lstio_dbg_nmlen + 1); + + return -EFAULT; + } + + name[args->lstio_dbg_nmlen] = 0; + } + + rc = -EINVAL; + + switch (args->lstio_dbg_type) { + case LST_OPC_SESSION: + rc = lstcon_session_debug(args->lstio_dbg_timeout, + args->lstio_dbg_resultp); + break; + + case LST_OPC_BATCHSRV: + client = 0; + case LST_OPC_BATCHCLI: + if (name == NULL) + goto out; + + rc = lstcon_batch_debug(args->lstio_dbg_timeout, + name, client, args->lstio_dbg_resultp); + break; + + case LST_OPC_GROUP: + if (name == NULL) + goto out; + + rc = lstcon_group_debug(args->lstio_dbg_timeout, + name, args->lstio_dbg_resultp); + break; + + case LST_OPC_NODES: + if (args->lstio_dbg_count <= 0 || + args->lstio_dbg_idsp == NULL) + goto out; + + rc = lstcon_nodes_debug(args->lstio_dbg_timeout, + args->lstio_dbg_count, + args->lstio_dbg_idsp, + args->lstio_dbg_resultp); + break; + + default: + break; + } + +out: + if (name != NULL) + LIBCFS_FREE(name, args->lstio_dbg_nmlen + 1); + + return rc; +} + +int +lst_group_add_ioctl(lstio_group_add_args_t *args) +{ + char *name; + int rc; + + if (args->lstio_grp_key != console_session.ses_key) + return -EACCES; + + if (args->lstio_grp_namep == NULL|| + args->lstio_grp_nmlen <= 0 || + args->lstio_grp_nmlen > LST_NAME_SIZE) + return -EINVAL; + + LIBCFS_ALLOC(name, args->lstio_grp_nmlen + 1); + if (name == NULL) + return -ENOMEM; + + if (copy_from_user(name, + args->lstio_grp_namep, + args->lstio_grp_nmlen)) { + LIBCFS_FREE(name, args->lstio_grp_nmlen); + return -EFAULT; + } + + name[args->lstio_grp_nmlen] = 0; + + rc = lstcon_group_add(name); + + LIBCFS_FREE(name, args->lstio_grp_nmlen + 1); + + return rc; +} + +int +lst_group_del_ioctl(lstio_group_del_args_t *args) +{ + int rc; + char *name; + + if (args->lstio_grp_key != console_session.ses_key) + return -EACCES; + + if (args->lstio_grp_namep == NULL || + args->lstio_grp_nmlen <= 0 || + args->lstio_grp_nmlen > LST_NAME_SIZE) + return -EINVAL; + + LIBCFS_ALLOC(name, args->lstio_grp_nmlen + 1); + if (name == NULL) + return -ENOMEM; + + if (copy_from_user(name, + args->lstio_grp_namep, + args->lstio_grp_nmlen)) { + LIBCFS_FREE(name, args->lstio_grp_nmlen + 1); + return -EFAULT; + } + + name[args->lstio_grp_nmlen] = 0; + + rc = lstcon_group_del(name); + + LIBCFS_FREE(name, args->lstio_grp_nmlen + 1); + + return rc; +} + +int +lst_group_update_ioctl(lstio_group_update_args_t *args) +{ + int rc; + char *name; + + if (args->lstio_grp_key != console_session.ses_key) + return -EACCES; + + if (args->lstio_grp_resultp == NULL || + args->lstio_grp_namep == NULL || + args->lstio_grp_nmlen <= 0 || + args->lstio_grp_nmlen > LST_NAME_SIZE) + return -EINVAL; + + LIBCFS_ALLOC(name, args->lstio_grp_nmlen + 1); + if (name == NULL) + return -ENOMEM; + + if (copy_from_user(name, + args->lstio_grp_namep, + args->lstio_grp_nmlen)) { + LIBCFS_FREE(name, args->lstio_grp_nmlen + 1); + return -EFAULT; + } + + name[args->lstio_grp_nmlen] = 0; + + switch (args->lstio_grp_opc) { + case LST_GROUP_CLEAN: + rc = lstcon_group_clean(name, args->lstio_grp_args); + break; + + case LST_GROUP_REFRESH: + rc = lstcon_group_refresh(name, args->lstio_grp_resultp); + break; + + case LST_GROUP_RMND: + if (args->lstio_grp_count <= 0 || + args->lstio_grp_idsp == NULL) { + rc = -EINVAL; + break; + } + rc = lstcon_nodes_remove(name, args->lstio_grp_count, + args->lstio_grp_idsp, + args->lstio_grp_resultp); + break; + + default: + rc = -EINVAL; + break; + } + + LIBCFS_FREE(name, args->lstio_grp_nmlen + 1); + + return rc; +} + +int +lst_nodes_add_ioctl(lstio_group_nodes_args_t *args) +{ + int rc; + char *name; + + if (args->lstio_grp_key != console_session.ses_key) + return -EACCES; + + if (args->lstio_grp_idsp == NULL || /* array of ids */ + args->lstio_grp_count <= 0 || + args->lstio_grp_resultp == NULL || + args->lstio_grp_namep == NULL || + args->lstio_grp_nmlen <= 0 || + args->lstio_grp_nmlen > LST_NAME_SIZE) + return -EINVAL; + + LIBCFS_ALLOC(name, args->lstio_grp_nmlen + 1); + if (name == NULL) + return -ENOMEM; + + if (copy_from_user(name, args->lstio_grp_namep, + args->lstio_grp_nmlen)) { + LIBCFS_FREE(name, args->lstio_grp_nmlen + 1); + + return -EFAULT; + } + + name[args->lstio_grp_nmlen] = 0; + + rc = lstcon_nodes_add(name, args->lstio_grp_count, + args->lstio_grp_idsp, + args->lstio_grp_resultp); + + LIBCFS_FREE(name, args->lstio_grp_nmlen + 1); + + return rc; +} + +int +lst_group_list_ioctl(lstio_group_list_args_t *args) +{ + if (args->lstio_grp_key != console_session.ses_key) + return -EACCES; + + if (args->lstio_grp_idx < 0 || + args->lstio_grp_namep == NULL || + args->lstio_grp_nmlen <= 0 || + args->lstio_grp_nmlen > LST_NAME_SIZE) + return -EINVAL; + + return lstcon_group_list(args->lstio_grp_idx, + args->lstio_grp_nmlen, + args->lstio_grp_namep); +} + +int +lst_group_info_ioctl(lstio_group_info_args_t *args) +{ + char *name; + int ndent; + int index; + int rc; + + if (args->lstio_grp_key != console_session.ses_key) + return -EACCES; + + if (args->lstio_grp_namep == NULL || + args->lstio_grp_nmlen <= 0 || + args->lstio_grp_nmlen > LST_NAME_SIZE) + return -EINVAL; + + if (args->lstio_grp_entp == NULL && /* output: group entry */ + args->lstio_grp_dentsp == NULL) /* output: node entry */ + return -EINVAL; + + if (args->lstio_grp_dentsp != NULL) { /* have node entry */ + if (args->lstio_grp_idxp == NULL || /* node index */ + args->lstio_grp_ndentp == NULL) /* # of node entry */ + return -EINVAL; + + if (copy_from_user(&ndent, + args->lstio_grp_ndentp, sizeof(ndent)) || + copy_from_user(&index, args->lstio_grp_idxp, sizeof(index))) + return -EFAULT; + + if (ndent <= 0 || index < 0) + return -EINVAL; + } + + LIBCFS_ALLOC(name, args->lstio_grp_nmlen + 1); + if (name == NULL) + return -ENOMEM; + + if (copy_from_user(name, + args->lstio_grp_namep, + args->lstio_grp_nmlen)) { + LIBCFS_FREE(name, args->lstio_grp_nmlen + 1); + return -EFAULT; + } + + name[args->lstio_grp_nmlen] = 0; + + rc = lstcon_group_info(name, args->lstio_grp_entp, + &index, &ndent, args->lstio_grp_dentsp); + + LIBCFS_FREE(name, args->lstio_grp_nmlen + 1); + + if (rc != 0) + return rc; + + if (args->lstio_grp_dentsp != NULL && + (copy_to_user(args->lstio_grp_idxp, &index, sizeof(index)) || + copy_to_user(args->lstio_grp_ndentp, &ndent, sizeof(ndent)))) + rc = -EFAULT; + + return 0; +} + +int +lst_batch_add_ioctl(lstio_batch_add_args_t *args) +{ + int rc; + char *name; + + if (args->lstio_bat_key != console_session.ses_key) + return -EACCES; + + if (args->lstio_bat_namep == NULL || + args->lstio_bat_nmlen <= 0 || + args->lstio_bat_nmlen > LST_NAME_SIZE) + return -EINVAL; + + LIBCFS_ALLOC(name, args->lstio_bat_nmlen + 1); + if (name == NULL) + return -ENOMEM; + + if (copy_from_user(name, + args->lstio_bat_namep, + args->lstio_bat_nmlen)) { + LIBCFS_FREE(name, args->lstio_bat_nmlen + 1); + return -EFAULT; + } + + name[args->lstio_bat_nmlen] = 0; + + rc = lstcon_batch_add(name); + + LIBCFS_FREE(name, args->lstio_bat_nmlen + 1); + + return rc; +} + +int +lst_batch_run_ioctl(lstio_batch_run_args_t *args) +{ + int rc; + char *name; + + if (args->lstio_bat_key != console_session.ses_key) + return -EACCES; + + if (args->lstio_bat_namep == NULL || + args->lstio_bat_nmlen <= 0 || + args->lstio_bat_nmlen > LST_NAME_SIZE) + return -EINVAL; + + LIBCFS_ALLOC(name, args->lstio_bat_nmlen + 1); + if (name == NULL) + return -ENOMEM; + + if (copy_from_user(name, + args->lstio_bat_namep, + args->lstio_bat_nmlen)) { + LIBCFS_FREE(name, args->lstio_bat_nmlen + 1); + return -EFAULT; + } + + name[args->lstio_bat_nmlen] = 0; + + rc = lstcon_batch_run(name, args->lstio_bat_timeout, + args->lstio_bat_resultp); + + LIBCFS_FREE(name, args->lstio_bat_nmlen + 1); + + return rc; +} + +int +lst_batch_stop_ioctl(lstio_batch_stop_args_t *args) +{ + int rc; + char *name; + + if (args->lstio_bat_key != console_session.ses_key) + return -EACCES; + + if (args->lstio_bat_resultp == NULL || + args->lstio_bat_namep == NULL || + args->lstio_bat_nmlen <= 0 || + args->lstio_bat_nmlen > LST_NAME_SIZE) + return -EINVAL; + + LIBCFS_ALLOC(name, args->lstio_bat_nmlen + 1); + if (name == NULL) + return -ENOMEM; + + if (copy_from_user(name, + args->lstio_bat_namep, + args->lstio_bat_nmlen)) { + LIBCFS_FREE(name, args->lstio_bat_nmlen + 1); + return -EFAULT; + } + + name[args->lstio_bat_nmlen] = 0; + + rc = lstcon_batch_stop(name, args->lstio_bat_force, + args->lstio_bat_resultp); + + LIBCFS_FREE(name, args->lstio_bat_nmlen + 1); + + return rc; +} + +int +lst_batch_query_ioctl(lstio_batch_query_args_t *args) +{ + char *name; + int rc; + + if (args->lstio_bat_key != console_session.ses_key) + return -EACCES; + + if (args->lstio_bat_resultp == NULL || + args->lstio_bat_namep == NULL || + args->lstio_bat_nmlen <= 0 || + args->lstio_bat_nmlen > LST_NAME_SIZE) + return -EINVAL; + + if (args->lstio_bat_testidx < 0) + return -EINVAL; + + LIBCFS_ALLOC(name, args->lstio_bat_nmlen + 1); + if (name == NULL) + return -ENOMEM; + + if (copy_from_user(name, + args->lstio_bat_namep, + args->lstio_bat_nmlen)) { + LIBCFS_FREE(name, args->lstio_bat_nmlen + 1); + return -EFAULT; + } + + name[args->lstio_bat_nmlen] = 0; + + rc = lstcon_test_batch_query(name, + args->lstio_bat_testidx, + args->lstio_bat_client, + args->lstio_bat_timeout, + args->lstio_bat_resultp); + + LIBCFS_FREE(name, args->lstio_bat_nmlen + 1); + + return rc; +} + +int +lst_batch_list_ioctl(lstio_batch_list_args_t *args) +{ + if (args->lstio_bat_key != console_session.ses_key) + return -EACCES; + + if (args->lstio_bat_idx < 0 || + args->lstio_bat_namep == NULL || + args->lstio_bat_nmlen <= 0 || + args->lstio_bat_nmlen > LST_NAME_SIZE) + return -EINVAL; + + return lstcon_batch_list(args->lstio_bat_idx, + args->lstio_bat_nmlen, + args->lstio_bat_namep); +} + +int +lst_batch_info_ioctl(lstio_batch_info_args_t *args) +{ + char *name; + int rc; + int index; + int ndent; + + if (args->lstio_bat_key != console_session.ses_key) + return -EACCES; + + if (args->lstio_bat_namep == NULL || /* batch name */ + args->lstio_bat_nmlen <= 0 || + args->lstio_bat_nmlen > LST_NAME_SIZE) + return -EINVAL; + + if (args->lstio_bat_entp == NULL && /* output: batch entry */ + args->lstio_bat_dentsp == NULL) /* output: node entry */ + return -EINVAL; + + if (args->lstio_bat_dentsp != NULL) { /* have node entry */ + if (args->lstio_bat_idxp == NULL || /* node index */ + args->lstio_bat_ndentp == NULL) /* # of node entry */ + return -EINVAL; + + if (copy_from_user(&index, args->lstio_bat_idxp, sizeof(index)) || + copy_from_user(&ndent, args->lstio_bat_ndentp, sizeof(ndent))) + return -EFAULT; + + if (ndent <= 0 || index < 0) + return -EINVAL; + } + + LIBCFS_ALLOC(name, args->lstio_bat_nmlen + 1); + if (name == NULL) + return -ENOMEM; + + if (copy_from_user(name, + args->lstio_bat_namep, args->lstio_bat_nmlen)) { + LIBCFS_FREE(name, args->lstio_bat_nmlen + 1); + return -EFAULT; + } + + name[args->lstio_bat_nmlen] = 0; + + rc = lstcon_batch_info(name, + args->lstio_bat_entp, args->lstio_bat_server, + args->lstio_bat_testidx, &index, &ndent, + args->lstio_bat_dentsp); + + LIBCFS_FREE(name, args->lstio_bat_nmlen + 1); + + if (rc != 0) + return rc; + + if (args->lstio_bat_dentsp != NULL && + (copy_to_user(args->lstio_bat_idxp, &index, sizeof(index)) || + copy_to_user(args->lstio_bat_ndentp, &ndent, sizeof(ndent)))) + rc = -EFAULT; + + return rc; +} + +int +lst_stat_query_ioctl(lstio_stat_args_t *args) +{ + int rc; + char *name; + + /* TODO: not finished */ + if (args->lstio_sta_key != console_session.ses_key) + return -EACCES; + + if (args->lstio_sta_resultp == NULL || + (args->lstio_sta_namep == NULL && + args->lstio_sta_idsp == NULL) || + args->lstio_sta_nmlen <= 0 || + args->lstio_sta_nmlen > LST_NAME_SIZE) + return -EINVAL; + + if (args->lstio_sta_idsp != NULL && + args->lstio_sta_count <= 0) + return -EINVAL; + + LIBCFS_ALLOC(name, args->lstio_sta_nmlen + 1); + if (name == NULL) + return -ENOMEM; + + if (copy_from_user(name, args->lstio_sta_namep, + args->lstio_sta_nmlen)) { + LIBCFS_FREE(name, args->lstio_sta_nmlen + 1); + return -EFAULT; + } + + if (args->lstio_sta_idsp == NULL) { + rc = lstcon_group_stat(name, args->lstio_sta_timeout, + args->lstio_sta_resultp); + } else { + rc = lstcon_nodes_stat(args->lstio_sta_count, + args->lstio_sta_idsp, + args->lstio_sta_timeout, + args->lstio_sta_resultp); + } + + LIBCFS_FREE(name, args->lstio_sta_nmlen + 1); + + return rc; +} + +int lst_test_add_ioctl(lstio_test_args_t *args) +{ + char *name; + char *srcgrp = NULL; + char *dstgrp = NULL; + void *param = NULL; + int rc = -ENOMEM; + + if (args->lstio_tes_resultp == NULL || + args->lstio_tes_bat_name == NULL || /* no specified batch */ + args->lstio_tes_bat_nmlen <= 0 || + args->lstio_tes_bat_nmlen > LST_NAME_SIZE || + args->lstio_tes_sgrp_name == NULL || /* no source group */ + args->lstio_tes_sgrp_nmlen <= 0 || + args->lstio_tes_sgrp_nmlen > LST_NAME_SIZE || + args->lstio_tes_dgrp_name == NULL || /* no target group */ + args->lstio_tes_dgrp_nmlen <= 0 || + args->lstio_tes_dgrp_nmlen > LST_NAME_SIZE) + return -EINVAL; + + /* have parameter, check if parameter length is valid */ + if (args->lstio_tes_param != NULL && + (args->lstio_tes_param_len <= 0 || + args->lstio_tes_param_len > CFS_PAGE_SIZE - sizeof(lstcon_test_t))) + return -EINVAL; + + LIBCFS_ALLOC(name, args->lstio_tes_bat_nmlen + 1); + if (name == NULL) + return rc; + + LIBCFS_ALLOC(srcgrp, args->lstio_tes_sgrp_nmlen + 1); + if (srcgrp == NULL) + goto out; + + LIBCFS_ALLOC(dstgrp, args->lstio_tes_dgrp_nmlen + 1); + if (srcgrp == NULL) + goto out; + + if (args->lstio_tes_param != NULL) { + LIBCFS_ALLOC(param, args->lstio_tes_param_len); + if (param == NULL) + goto out; + } + + rc = -EFAULT; + if (copy_from_user(name, + args->lstio_tes_bat_name, + args->lstio_tes_bat_nmlen) || + copy_from_user(srcgrp, + args->lstio_tes_sgrp_name, + args->lstio_tes_sgrp_nmlen) || + copy_from_user(dstgrp, + args->lstio_tes_dgrp_name, + args->lstio_tes_dgrp_nmlen) || + copy_from_user(param, args->lstio_tes_param, + args->lstio_tes_param_len)) + goto out; + + rc = lstcon_test_add(name, + args->lstio_tes_type, + args->lstio_tes_loop, + args->lstio_tes_concur, + args->lstio_tes_dist, args->lstio_tes_span, + srcgrp, dstgrp, param, args->lstio_tes_param_len, + args->lstio_tes_resultp); +out: + if (name != NULL) + LIBCFS_FREE(name, args->lstio_tes_bat_nmlen + 1); + + if (srcgrp != NULL) + LIBCFS_FREE(srcgrp, args->lstio_tes_sgrp_nmlen + 1); + + if (dstgrp != NULL) + LIBCFS_FREE(dstgrp, args->lstio_tes_dgrp_nmlen + 1); + + if (param != NULL) + LIBCFS_FREE(param, args->lstio_tes_param_len); + + return rc; +} + +int +lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_data *data) +{ + char *buf; + int opc = data->ioc_u32[0]; + int rc; + + if (cmd != IOC_LIBCFS_LNETST) + return -EINVAL; + + if (data->ioc_plen1 > CFS_PAGE_SIZE) + return -EINVAL; + + LIBCFS_ALLOC(buf, data->ioc_plen1); + if (buf == NULL) + return -ENOMEM; + + /* copy in parameter */ + if (copy_from_user(buf, data->ioc_pbuf1, data->ioc_plen1)) { + LIBCFS_FREE(buf, data->ioc_plen1); + return -EFAULT; + } + + mutex_down(&console_session.ses_mutex); + + console_session.ses_laststamp = cfs_time_current_sec(); + + if (console_session.ses_shutdown) { + rc = -ESHUTDOWN; + goto out; + } + + if (console_session.ses_expired) + lstcon_session_end(); + + if (opc != LSTIO_SESSION_NEW && + console_session.ses_state == LST_SESSION_NONE) { + CDEBUG(D_NET, "LST no active session\n"); + rc = -ESRCH; + goto out; + } + + memset(&console_session.ses_trans_stat, 0, sizeof(lstcon_trans_stat_t)); + + switch (opc) { + case LSTIO_SESSION_NEW: + rc = lst_session_new_ioctl((lstio_session_new_args_t *)buf); + break; + case LSTIO_SESSION_END: + rc = lst_session_end_ioctl((lstio_session_end_args_t *)buf); + break; + case LSTIO_SESSION_INFO: + rc = lst_session_info_ioctl((lstio_session_info_args_t *)buf); + break; + case LSTIO_DEBUG: + rc = lst_debug_ioctl((lstio_debug_args_t *)buf); + break; + case LSTIO_GROUP_ADD: + rc = lst_group_add_ioctl((lstio_group_add_args_t *)buf); + break; + case LSTIO_GROUP_DEL: + rc = lst_group_del_ioctl((lstio_group_del_args_t *)buf); + break; + case LSTIO_GROUP_UPDATE: + rc = lst_group_update_ioctl((lstio_group_update_args_t *)buf); + break; + case LSTIO_NODES_ADD: + rc = lst_nodes_add_ioctl((lstio_group_nodes_args_t *)buf); + break; + case LSTIO_GROUP_LIST: + rc = lst_group_list_ioctl((lstio_group_list_args_t *)buf); + break; + case LSTIO_GROUP_INFO: + rc = lst_group_info_ioctl((lstio_group_info_args_t *)buf); + break; + case LSTIO_BATCH_ADD: + rc = lst_batch_add_ioctl((lstio_batch_add_args_t *)buf); + break; + case LSTIO_BATCH_START: + rc = lst_batch_run_ioctl((lstio_batch_run_args_t *)buf); + break; + case LSTIO_BATCH_STOP: + rc = lst_batch_stop_ioctl((lstio_batch_stop_args_t *)buf); + break; + case LSTIO_BATCH_QUERY: + rc = lst_batch_query_ioctl((lstio_batch_query_args_t *)buf); + break; + case LSTIO_BATCH_LIST: + rc = lst_batch_list_ioctl((lstio_batch_list_args_t *)buf); + break; + case LSTIO_BATCH_INFO: + rc = lst_batch_info_ioctl((lstio_batch_info_args_t *)buf); + break; + case LSTIO_TEST_ADD: + rc = lst_test_add_ioctl((lstio_test_args_t *)buf); + break; + case LSTIO_STAT_QUERY: + rc = lst_stat_query_ioctl((lstio_stat_args_t *)buf); + break; + default: + rc = -EINVAL; + } + + if (copy_to_user(data->ioc_pbuf2, &console_session.ses_trans_stat, + sizeof(lstcon_trans_stat_t))) + rc = -EFAULT; +out: + mutex_up(&console_session.ses_mutex); + + LIBCFS_FREE(buf, data->ioc_plen1); + + return rc; +} + +EXPORT_SYMBOL(lstcon_ioctl_entry); + +#endif diff --git a/lnet/selftest/conrpc.c b/lnet/selftest/conrpc.c new file mode 100644 index 0000000..028bdc1 --- /dev/null +++ b/lnet/selftest/conrpc.c @@ -0,0 +1,1293 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Author: Liang Zhen + * + * This file is part of Lustre, http://www.lustre.org + * + * Console framework rpcs + */ +#ifdef __KERNEL__ + +#include +#include +#include "timer.h" +#include "conrpc.h" +#include "console.h" + +void lstcon_rpc_stat_reply(int, srpc_msg_t *, + lstcon_node_t *, lstcon_trans_stat_t *); + +static void +lstcon_rpc_done(srpc_client_rpc_t *rpc) +{ + lstcon_rpc_t *crpc = (lstcon_rpc_t *)rpc->crpc_priv; + + LASSERT (!list_empty(&crpc->crp_link)); + LASSERT (crpc != NULL && rpc == crpc->crp_rpc); + LASSERT (crpc->crp_posted && !crpc->crp_finished); + + spin_lock(&rpc->crpc_lock); + + if (crpc->crp_trans == NULL) { + /* orphan RPC */ + spin_lock(&console_session.ses_rpc_lock); + + /* delete from orphan rpcs list */ + console_session.ses_rpc_pending --; + list_del_init(&crpc->crp_link); + + spin_unlock(&console_session.ses_rpc_lock); + + spin_unlock(&rpc->crpc_lock); + + /* release it */ + lstcon_rpc_put(crpc); + return; + } + + /* not an orphan RPC */ + crpc->crp_finished = 1; + + if (crpc->crp_stamp == 0) { + /* not aborted */ + LASSERT (crpc->crp_status == 0); + + crpc->crp_stamp = cfs_time_current(); + crpc->crp_status = rpc->crpc_status; + } + + /* wakeup thread waiting on the group if + * it's the last rpc in the group */ + if (atomic_dec_and_test(&crpc->crp_trans->tas_remaining)) + cfs_waitq_signal(&crpc->crp_trans->tas_waitq); + + spin_unlock(&rpc->crpc_lock); +} + +int +lstcon_rpc_init(lstcon_node_t *nd, int service, + int npg, int cached, lstcon_rpc_t *crpc) +{ + + crpc->crp_rpc = sfw_create_rpc(nd->nd_id, service, + npg, npg * CFS_PAGE_SIZE, + lstcon_rpc_done, (void *)crpc); + if (crpc->crp_rpc == NULL) + return -ENOMEM; + + crpc->crp_trans = NULL; + crpc->crp_node = nd; + crpc->crp_posted = 0; + crpc->crp_finished = 0; + crpc->crp_unpacked = 0; + crpc->crp_status = 0; + crpc->crp_stamp = 0; + crpc->crp_static = !cached; + CFS_INIT_LIST_HEAD(&crpc->crp_link); + + return 0; +} + +int +lstcon_rpc_prep(lstcon_node_t *nd, int service, + int npg, lstcon_rpc_t **crpcpp) +{ + lstcon_rpc_t *crpc = NULL; + int rc; + + spin_lock(&console_session.ses_rpc_lock); + + if (!list_empty(&console_session.ses_rpc_freelist)) { + crpc = list_entry(console_session.ses_rpc_freelist.next, + lstcon_rpc_t, crp_link); + list_del(&crpc->crp_link); + } + + spin_unlock(&console_session.ses_rpc_lock); + + if (crpc == NULL) { + LIBCFS_ALLOC(crpc, sizeof(*crpc)); + if (crpc == NULL) + return -ENOMEM; + } + + rc = lstcon_rpc_init(nd, service, npg, 1, crpc); + if (rc == 0) { + *crpcpp = crpc; + return 0; + } + + LIBCFS_FREE(crpc, sizeof(*crpc)); + + return rc; +} + +void +lstcon_rpc_put(lstcon_rpc_t *crpc) +{ + srpc_bulk_t *bulk = &crpc->crp_rpc->crpc_bulk; + int i; + + LASSERT (list_empty(&crpc->crp_link)); + + for (i = 0; i < bulk->bk_niov; i++) { + if (bulk->bk_iovs[i].kiov_page == NULL) + continue; + + cfs_free_page(bulk->bk_iovs[i].kiov_page); + } + + srpc_client_rpc_decref(crpc->crp_rpc); + + if (crpc->crp_static) { + memset(crpc, 0, sizeof(*crpc)); + crpc->crp_static = 1; + return; + } + + spin_lock(&console_session.ses_rpc_lock); + + list_add(&crpc->crp_link, &console_session.ses_rpc_freelist); + + spin_unlock(&console_session.ses_rpc_lock); +} + +void +lstcon_rpc_post(lstcon_rpc_t *crpc) +{ + lstcon_rpc_trans_t *trans = crpc->crp_trans; + + LASSERT (trans != NULL); + + atomic_inc(&trans->tas_remaining); + crpc->crp_posted = 1; + + sfw_post_rpc(crpc->crp_rpc); +} + +static char * +lstcon_rpc_trans_name(int transop) +{ + if (transop == LST_TRANS_SESNEW) + return "SESNEW"; + + if (transop == LST_TRANS_SESEND) + return "SESEND"; + + if (transop == LST_TRANS_SESQRY) + return "SESQRY"; + + if (transop == LST_TRANS_SESPING) + return "SESPING"; + + if (transop == LST_TRANS_TSBCLIADD) + return "TSBCLIADD"; + + if (transop == LST_TRANS_TSBSRVADD) + return "TSBSRVADD"; + + if (transop == LST_TRANS_TSBRUN) + return "TSBRUN"; + + if (transop == LST_TRANS_TSBSTOP) + return "TSBSTOP"; + + if (transop == LST_TRANS_TSBCLIQRY) + return "TSBCLIQRY"; + + if (transop == LST_TRANS_TSBSRVQRY) + return "TSBSRVQRY"; + + if (transop == LST_TRANS_STATQRY) + return "STATQRY"; + + return "Unknown"; +} + +int +lstcon_rpc_trans_prep(struct list_head *translist, + int transop, lstcon_rpc_trans_t **transpp) +{ + lstcon_rpc_trans_t *trans; + + if (translist != NULL) { + list_for_each_entry(trans, translist, tas_link) { + /* Can't enqueue two private transaction on + * the same object */ + if ((trans->tas_opc & transop) == LST_TRANS_PRIVATE) + return -EPERM; + } + } + + /* create a trans group */ + LIBCFS_ALLOC(trans, sizeof(*trans)); + if (trans == NULL) + return -ENOMEM; + + trans->tas_opc = transop; + + if (translist == NULL) + CFS_INIT_LIST_HEAD(&trans->tas_olink); + else + list_add_tail(&trans->tas_olink, translist); + + list_add_tail(&trans->tas_link, &console_session.ses_trans_list); + + CFS_INIT_LIST_HEAD(&trans->tas_rpcs_list); + atomic_set(&trans->tas_remaining, 0); + cfs_waitq_init(&trans->tas_waitq); + + *transpp = trans; + + return 0; +} + +void +lstcon_rpc_trans_addreq(lstcon_rpc_trans_t *trans, lstcon_rpc_t *crpc) +{ + list_add_tail(&crpc->crp_link, &trans->tas_rpcs_list); + crpc->crp_trans = trans; +} + +void +lstcon_rpc_trans_abort(lstcon_rpc_trans_t *trans, int error) +{ + srpc_client_rpc_t *rpc; + lstcon_rpc_t *crpc; + lstcon_node_t *nd; + + list_for_each_entry (crpc, &trans->tas_rpcs_list, crp_link) { + rpc = crpc->crp_rpc; + + spin_lock(&rpc->crpc_lock); + + if (!crpc->crp_posted || crpc->crp_stamp != 0) { + /* rpc done or aborted already */ + spin_unlock(&rpc->crpc_lock); + continue; + } + + crpc->crp_stamp = cfs_time_current(); + crpc->crp_status = error; + + spin_unlock(&rpc->crpc_lock); + + sfw_abort_rpc(rpc); + + if (error != ETIMEDOUT) + continue; + + nd = crpc->crp_node; + if (cfs_time_after(nd->nd_stamp, crpc->crp_stamp)) + continue; + + nd->nd_stamp = crpc->crp_stamp; + nd->nd_state = LST_NODE_DOWN; + } +} + +static int +lstcon_rpc_trans_check(lstcon_rpc_trans_t *trans) +{ + if (console_session.ses_shutdown && + !list_empty(&trans->tas_olink)) /* It's not an end session RPC */ + return 1; + + return (atomic_read(&trans->tas_remaining) == 0) ? 1: 0; +} + +int +lstcon_rpc_trans_postwait(lstcon_rpc_trans_t *trans, int timeout) +{ + lstcon_rpc_t *crpc; + int rc; + + if (list_empty(&trans->tas_rpcs_list)) + return 0; + + if (timeout < LST_TRANS_MIN_TIMEOUT) + timeout = LST_TRANS_MIN_TIMEOUT; + + CDEBUG(D_NET, "Transaction %s started\n", + lstcon_rpc_trans_name(trans->tas_opc)); + + /* post all requests */ + list_for_each_entry (crpc, &trans->tas_rpcs_list, crp_link) { + LASSERT (!crpc->crp_posted); + + lstcon_rpc_post(crpc); + } + + mutex_up(&console_session.ses_mutex); + + rc = wait_event_interruptible_timeout(trans->tas_waitq, + lstcon_rpc_trans_check(trans), + timeout * HZ); + + rc = (rc > 0)? 0: ((rc < 0)? -EINTR: -ETIMEDOUT); + + mutex_down(&console_session.ses_mutex); + + if (console_session.ses_shutdown) + rc = -ESHUTDOWN; + + if (rc != 0) { + /* treat short timeout as canceled */ + if (rc == -ETIMEDOUT && timeout < LST_TRANS_MIN_TIMEOUT * 2) + rc = -EINTR; + + lstcon_rpc_trans_abort(trans, rc); + } + + CDEBUG(D_NET, "Transaction %s stopped: %d\n", + lstcon_rpc_trans_name(trans->tas_opc), rc); + + lstcon_rpc_trans_stat(trans, lstcon_trans_stat()); + + return rc; +} + +int +lstcon_rpc_get_reply(lstcon_rpc_t *crpc, srpc_msg_t **msgpp) +{ + lstcon_node_t *nd = crpc->crp_node; + srpc_client_rpc_t *rpc = crpc->crp_rpc; + srpc_generic_reply_t *rep; + + LASSERT (nd != NULL && rpc != NULL); + LASSERT (crpc->crp_stamp != 0); + + if (crpc->crp_status != 0) { + *msgpp = NULL; + return crpc->crp_status; + } + + *msgpp = &rpc->crpc_replymsg; + if (!crpc->crp_unpacked) { + sfw_unpack_message(*msgpp); + crpc->crp_unpacked = 1; + } + + if (cfs_time_after(nd->nd_stamp, crpc->crp_stamp)) + return 0; + + nd->nd_stamp = crpc->crp_stamp; + rep = &(*msgpp)->msg_body.reply; + + if (rep->sid.ses_nid == LNET_NID_ANY) + nd->nd_state = LST_NODE_UNKNOWN; + else if (lstcon_session_match(rep->sid)) + nd->nd_state = LST_NODE_ACTIVE; + else + nd->nd_state = LST_NODE_BUSY; + + return 0; +} + +void +lstcon_rpc_trans_stat(lstcon_rpc_trans_t *trans, lstcon_trans_stat_t *stat) +{ + lstcon_rpc_t *crpc; + srpc_client_rpc_t *rpc; + srpc_msg_t *rep; + int error; + + LASSERT (stat != NULL); + + memset(stat, 0, sizeof(*stat)); + + list_for_each_entry(crpc, &trans->tas_rpcs_list, crp_link) { + lstcon_rpc_stat_total(stat, 1); + + rpc = crpc->crp_rpc; + + LASSERT (crpc->crp_stamp != 0); + + error = lstcon_rpc_get_reply(crpc, &rep); + if (error != 0) { + lstcon_rpc_stat_failure(stat, 1); + if (stat->trs_rpc_errno == 0) + stat->trs_rpc_errno = -error; + + continue; + } + + lstcon_rpc_stat_success(stat, 1); + + lstcon_rpc_stat_reply(trans->tas_opc, rep, + crpc->crp_node, stat); + } + + CDEBUG(D_NET, "transaction %s success, %d failure, %d total %d, " + "RPC error(%d), Framework error(%d)\n", + lstcon_rpc_trans_name(trans->tas_opc), + lstcon_rpc_stat_success(stat, 0), + lstcon_rpc_stat_failure(stat, 0), + lstcon_rpc_stat_total(stat, 0), + stat->trs_rpc_errno, stat->trs_fwk_errno); + + return; +} + +int +lstcon_rpc_trans_interpreter(lstcon_rpc_trans_t *trans, + struct list_head *head_up, + lstcon_rpc_readent_func_t readent) +{ + struct list_head tmp; + struct list_head *next; + lstcon_rpc_ent_t *ent; + srpc_generic_reply_t *rep; + srpc_client_rpc_t *rpc; + lstcon_rpc_t *crpc; + srpc_msg_t *msg; + lstcon_node_t *nd; + cfs_duration_t dur; + struct timeval tv; + int error; + + LASSERT (head_up != NULL); + + next = head_up; + + list_for_each_entry(crpc, &trans->tas_rpcs_list, crp_link) { + if (copy_from_user(&tmp, next, sizeof(struct list_head))) + return -EFAULT; + + if (tmp.next == head_up) + return 0; + + next = tmp.next; + + ent = list_entry(next, lstcon_rpc_ent_t, rpe_link); + + rpc = crpc->crp_rpc; + + LASSERT (crpc->crp_stamp != 0); + + error = lstcon_rpc_get_reply(crpc, &msg); + + nd = crpc->crp_node; + + dur = cfs_time_sub(crpc->crp_stamp, + console_session.ses_id.ses_stamp); + cfs_duration_usec(dur, &tv); + + if (copy_to_user(&ent->rpe_peer, + &nd->nd_id, sizeof(lnet_process_id_t)) || + copy_to_user(&ent->rpe_stamp, &tv, sizeof(tv)) || + copy_to_user(&ent->rpe_state, + &nd->nd_state, sizeof(nd->nd_state)) || + copy_to_user(&ent->rpe_rpc_errno, &error, sizeof(error))) + return -EFAULT; + + if (error != 0) + continue; + + /* RPC is done */ + rep = (srpc_generic_reply_t *)&msg->msg_body.reply; + + if (copy_to_user(&ent->rpe_sid, + &rep->sid, sizeof(lst_sid_t)) || + copy_to_user(&ent->rpe_fwk_errno, + &rep->status, sizeof(rep->status))) + return -EFAULT; + + if (readent == NULL) + continue; + + if ((error = readent(trans->tas_opc, msg, ent)) != 0) + return error; + } + + return 0; +} + +void +lstcon_rpc_trans_destroy(lstcon_rpc_trans_t *trans) +{ + srpc_client_rpc_t *rpc; + lstcon_rpc_t *crpc; + lstcon_rpc_t *tmp; + int count = 0; + + list_for_each_entry_safe(crpc, tmp, + &trans->tas_rpcs_list, crp_link) { + rpc = crpc->crp_rpc; + + spin_lock(&rpc->crpc_lock); + + /* free it if not posted or finished already */ + if (!crpc->crp_posted || crpc->crp_finished) { + spin_unlock(&rpc->crpc_lock); + + list_del_init(&crpc->crp_link); + lstcon_rpc_put(crpc); + + continue; + } + + /* rpcs can be still not callbacked (even LNetMDUnlink is called) + * because huge timeout for inaccessible network, don't make + * user wait for them, just put rpcs in orphan list */ + + LASSERT (crpc->crp_status != 0); + + crpc->crp_node = NULL; + crpc->crp_trans = NULL; + list_del(&crpc->crp_link); + + spin_lock(&console_session.ses_rpc_lock); + + count ++; + /* add to orphan list */ + console_session.ses_rpc_pending ++; + list_add_tail(&crpc->crp_link, &console_session.ses_rpc_list); + + spin_unlock(&console_session.ses_rpc_lock); + + spin_unlock(&rpc->crpc_lock); + + atomic_dec(&trans->tas_remaining); + } + + LASSERT (atomic_read(&trans->tas_remaining) == 0); + + list_del(&trans->tas_link); + if (!list_empty(&trans->tas_olink)) + list_del(&trans->tas_olink); + + CDEBUG(D_NET, "Transaction %s destroyed with %d pending RPCs\n", + lstcon_rpc_trans_name(trans->tas_opc), count); + + LIBCFS_FREE(trans, sizeof(*trans)); + + return; +} + +int +lstcon_sesrpc_prep(lstcon_node_t *nd, int transop, lstcon_rpc_t **crpc) +{ + srpc_mksn_reqst_t *msrq; + srpc_rmsn_reqst_t *rsrq; + int rc; + + switch (transop) { + case LST_TRANS_SESNEW: + rc = lstcon_rpc_prep(nd, SRPC_SERVICE_MAKE_SESSION, 0, crpc); + if (rc != 0) + return rc; + + msrq = &(*crpc)->crp_rpc->crpc_reqstmsg.msg_body.mksn_reqst; + msrq->mksn_sid = console_session.ses_id; + msrq->mksn_force = console_session.ses_force; + strncpy(msrq->mksn_name, console_session.ses_name, + strlen(console_session.ses_name)); + break; + + case LST_TRANS_SESEND: + rc = lstcon_rpc_prep(nd, SRPC_SERVICE_REMOVE_SESSION, 0, crpc); + if (rc != 0) + return rc; + + rsrq = &(*crpc)->crp_rpc->crpc_reqstmsg.msg_body.rmsn_reqst; + rsrq->rmsn_sid = console_session.ses_id; + break; + + default: + LBUG(); + } + + return 0; +} + +int +lstcon_dbgrpc_prep(lstcon_node_t *nd, lstcon_rpc_t **crpc) +{ + srpc_debug_reqst_t *drq; + int rc; + + rc = lstcon_rpc_prep(nd, SRPC_SERVICE_DEBUG, 0, crpc); + if (rc != 0) + return rc; + + drq = &(*crpc)->crp_rpc->crpc_reqstmsg.msg_body.dbg_reqst; + + drq->dbg_sid = console_session.ses_id; + drq->dbg_flags = 0; + + return rc; +} + +int +lstcon_batrpc_prep(lstcon_node_t *nd, int transop, + lstcon_tsb_hdr_t *tsb, lstcon_rpc_t **crpc) +{ + lstcon_batch_t *batch; + srpc_batch_reqst_t *brq; + int rc; + + rc = lstcon_rpc_prep(nd, SRPC_SERVICE_BATCH, 0, crpc); + if (rc != 0) + return rc; + + brq = &(*crpc)->crp_rpc->crpc_reqstmsg.msg_body.bat_reqst; + + brq->bar_sid = console_session.ses_id; + brq->bar_bid = tsb->tsb_id; + brq->bar_testidx = tsb->tsb_index; + brq->bar_opc = transop == LST_TRANS_TSBRUN ? SRPC_BATCH_OPC_RUN : + (transop == LST_TRANS_TSBSTOP ? SRPC_BATCH_OPC_STOP: + SRPC_BATCH_OPC_QUERY); + + if (transop != LST_TRANS_TSBRUN && + transop != LST_TRANS_TSBSTOP) + return 0; + + LASSERT (tsb->tsb_index == 0); + + batch = (lstcon_batch_t *)tsb; + brq->bar_arg = batch->bat_arg; + + return 0; +} + +int +lstcon_statrpc_prep(lstcon_node_t *nd, lstcon_rpc_t **crpc) +{ + srpc_stat_reqst_t *srq; + int rc; + + rc = lstcon_rpc_prep(nd, SRPC_SERVICE_QUERY_STAT, 0, crpc); + if (rc != 0) + return rc; + + srq = &(*crpc)->crp_rpc->crpc_reqstmsg.msg_body.stat_reqst; + + srq->str_sid = console_session.ses_id; + srq->str_type = 0; /* XXX remove it */ + + return 0; +} + +lnet_process_id_t * +lstcon_next_id(int idx, int nkiov, lnet_kiov_t *kiov) +{ + lnet_process_id_t *pid; + int i; + + i = idx / (CFS_PAGE_SIZE / sizeof(lnet_process_id_t)); + + LASSERT (i < nkiov); + + pid = (lnet_process_id_t *)cfs_page_address(kiov[i].kiov_page); + + return &pid[idx % (CFS_PAGE_SIZE / sizeof(lnet_process_id_t))]; +} + +int +lstcon_dstnodes_prep(lstcon_group_t *grp, int idx, + int dist, int span, int nkiov, lnet_kiov_t *kiov) +{ + lnet_process_id_t *pid; + lstcon_ndlink_t *ndl; + lstcon_node_t *nd; + int start; + int end; + int i = 0; + + LASSERT (dist >= 1); + LASSERT (span >= 1); + LASSERT (grp->grp_nnode >= 1); + + if (span > grp->grp_nnode) + return -EINVAL; + + start = ((idx / dist) * span) % grp->grp_nnode; + end = ((idx / dist) * span + span - 1) % grp->grp_nnode; + + list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link) { + nd = ndl->ndl_node; + if (i < start) { + i ++; + continue; + } + + if (i > (end >= start ? end: grp->grp_nnode)) + break; + + pid = lstcon_next_id((i - start), nkiov, kiov); + *pid = nd->nd_id; + i++; + } + + if (start <= end) /* done */ + return 0; + + list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link) { + if (i > grp->grp_nnode + end) + break; + + nd = ndl->ndl_node; + pid = lstcon_next_id((i - start), nkiov, kiov); + *pid = nd->nd_id; + i++; + } + + return 0; +} + +int +lstcon_pingrpc_prep(lst_test_ping_param_t *param, srpc_test_reqst_t *req) +{ + test_ping_req_t *prq = &req->tsr_u.ping; + + prq->png_size = param->png_size; + prq->png_flags = param->png_flags; + /* TODO dest */ + return 0; +} + +int +lstcon_bulkrpc_prep(lst_test_bulk_param_t *param, srpc_test_reqst_t *req) +{ + test_bulk_req_t *brq = &req->tsr_u.bulk; + + brq->blk_opc = param->blk_opc; + brq->blk_npg = param->blk_npg; + brq->blk_flags = param->blk_flags; + + return 0; +} + +int +lstcon_testrpc_prep(lstcon_node_t *nd, int transop, + lstcon_test_t *test, lstcon_rpc_t **crpc) +{ + lstcon_group_t *sgrp = test->tes_src_grp; + lstcon_group_t *dgrp = test->tes_dst_grp; + srpc_test_reqst_t *trq; + srpc_bulk_t *bulk; + int i; + int n = 0; + int rc = 0; + + if (transop == LST_TRANS_TSBCLIADD) + n = sfw_id_pages(test->tes_span); + + rc = lstcon_rpc_prep(nd, SRPC_SERVICE_TEST, n, crpc); + if (rc != 0) + return rc; + + trq = &(*crpc)->crp_rpc->crpc_reqstmsg.msg_body.tes_reqst; + + if (transop == LST_TRANS_TSBSRVADD) { + int ndist = (sgrp->grp_nnode + test->tes_dist - 1) / test->tes_dist; + int nspan = (dgrp->grp_nnode + test->tes_span - 1) / test->tes_span; + int nmax = (ndist + nspan - 1) / nspan; + + trq->tsr_ndest = 0; + trq->tsr_loop = nmax * test->tes_dist * test->tes_concur; + + } else { + bulk = &(*crpc)->crp_rpc->crpc_bulk; + + for (i = 0; i < n; i++) { + bulk->bk_iovs[i].kiov_offset = 0; + bulk->bk_iovs[i].kiov_len = CFS_PAGE_SIZE; + bulk->bk_iovs[i].kiov_page = cfs_alloc_page(CFS_ALLOC_STD); + + if (bulk->bk_iovs[i].kiov_page != NULL) + continue; + + lstcon_rpc_put(*crpc); + return -ENOMEM; + } + + bulk->bk_sink = 0; + + LASSERT (transop == LST_TRANS_TSBCLIADD); + + rc = lstcon_dstnodes_prep(test->tes_dst_grp, + test->tes_cliidx++, test->tes_dist, + test->tes_span, n, &bulk->bk_iovs[0]); + if (rc != 0) { + lstcon_rpc_put(*crpc); + return rc; + } + + trq->tsr_ndest = test->tes_span; + trq->tsr_loop = test->tes_loop; + } + + trq->tsr_sid = console_session.ses_id; + trq->tsr_bid = test->tes_hdr.tsb_id; + trq->tsr_concur = test->tes_concur; + trq->tsr_is_client = (transop == LST_TRANS_TSBCLIADD) ? 1 : 0; + trq->tsr_stop_onerr = test->tes_stop_onerr; + + switch (test->tes_type) { + case LST_TEST_PING: + trq->tsr_service = SRPC_SERVICE_PING; + rc = lstcon_pingrpc_prep((lst_test_ping_param_t *)&test->tes_param[0], trq); + break; + case LST_TEST_BULK: + trq->tsr_service = SRPC_SERVICE_BRW; + rc = lstcon_bulkrpc_prep((lst_test_bulk_param_t *)&test->tes_param[0], trq); + break; + default: + LBUG(); + break; + } + + return rc; +} + +void +lstcon_rpc_stat_reply(int transop, srpc_msg_t *msg, + lstcon_node_t *nd, lstcon_trans_stat_t *stat) +{ + srpc_mksn_reply_t *mksn_rep; + srpc_rmsn_reply_t *rmsn_rep; + srpc_debug_reply_t *dbg_rep; + srpc_batch_reply_t *bat_rep; + srpc_test_reply_t *test_rep; + srpc_stat_reply_t *stat_rep; + int errno = 0; + + switch (transop) { + case LST_TRANS_SESNEW: + mksn_rep = &msg->msg_body.mksn_reply; + + if (mksn_rep->mksn_status == 0) { + lstcon_sesop_stat_success(stat, 1); + /* session timeout on remote node */ + nd->nd_timeout = mksn_rep->mksn_timeout; + return; + } + + LASSERT (mksn_rep->mksn_status == EBUSY || + mksn_rep->mksn_status == EINVAL); + + lstcon_sesop_stat_failure(stat, 1); + errno = mksn_rep->mksn_status; + break; + + case LST_TRANS_SESEND: + rmsn_rep = &msg->msg_body.rmsn_reply; + /* ESRCH is not an error for end session */ + if (rmsn_rep->rmsn_status == 0 || + rmsn_rep->rmsn_status == ESRCH) { + lstcon_sesop_stat_success(stat, 1); + return; + } + + LASSERT (rmsn_rep->rmsn_status == EBUSY || + rmsn_rep->rmsn_status == EINVAL); + + lstcon_sesop_stat_failure(stat, 1); + errno = rmsn_rep->rmsn_status; + break; + + case LST_TRANS_SESQRY: + case LST_TRANS_SESPING: + dbg_rep = &msg->msg_body.dbg_reply; + + if (dbg_rep->dbg_status == ESRCH) { + lstcon_sesqry_stat_unknown(stat, 1); + return; + } + + LASSERT (dbg_rep->dbg_status == 0); + + if (lstcon_session_match(dbg_rep->dbg_sid)) + lstcon_sesqry_stat_active(stat, 1); + else + lstcon_sesqry_stat_busy(stat, 1); + return; + + case LST_TRANS_TSBRUN: + case LST_TRANS_TSBSTOP: + bat_rep = &msg->msg_body.bat_reply; + + if (bat_rep->bar_status == 0) { + lstcon_tsbop_stat_success(stat, 1); + return; + } + + if (bat_rep->bar_status == EPERM && + transop == LST_TRANS_TSBSTOP) { + lstcon_tsbop_stat_success(stat, 1); + return; + } + + lstcon_tsbop_stat_failure(stat, 1); + errno = bat_rep->bar_status; + break; + + case LST_TRANS_TSBCLIQRY: + case LST_TRANS_TSBSRVQRY: + bat_rep = &msg->msg_body.bat_reply; + + if (bat_rep->bar_active != 0) + lstcon_tsbqry_stat_run(stat, 1); + else + lstcon_tsbqry_stat_idle(stat, 1); + + if (bat_rep->bar_status == 0) + return; + + lstcon_tsbqry_stat_failure(stat, 1); + errno = bat_rep->bar_status; + break; + + case LST_TRANS_TSBCLIADD: + case LST_TRANS_TSBSRVADD: + test_rep = &msg->msg_body.tes_reply; + + if (test_rep->tsr_status == 0) { + lstcon_tsbop_stat_success(stat, 1); + return; + } + + lstcon_tsbop_stat_failure(stat, 1); + errno = test_rep->tsr_status; + break; + + case LST_TRANS_STATQRY: + stat_rep = &msg->msg_body.stat_reply; + + if (stat_rep->str_status == 0) { + lstcon_statqry_stat_success(stat, 1); + return; + } + + lstcon_statqry_stat_failure(stat, 1); + errno = stat_rep->str_status; + break; + + default: + LBUG(); + } + + if (stat->trs_fwk_errno == 0) + stat->trs_fwk_errno = errno; + + return; +} + +int +lstcon_rpc_trans_ndlist(struct list_head *ndlist, + struct list_head *translist, int transop, void *arg, + lstcon_rpc_cond_func_t condition, lstcon_rpc_trans_t **transpp) +{ + lstcon_rpc_trans_t *trans; + lstcon_ndlink_t *ndl; + lstcon_node_t *nd; + lstcon_rpc_t *rpc; + int rc; + + /* Creating session RPG for list of nodes */ + + rc = lstcon_rpc_trans_prep(translist, transop, &trans); + if (rc != 0) { + CERROR("Can't create transaction %d: %d\n", transop, rc); + return rc; + } + + list_for_each_entry(ndl, ndlist, ndl_link) { + rc = condition == NULL ? 1 : + condition(transop, ndl->ndl_node, arg); + + if (rc == 0) + continue; + + if (rc < 0) { + CDEBUG(D_NET, "Condition error while creating RPC " + " for transaction %d: %d\n", transop, rc); + break; + } + + nd = ndl->ndl_node; + + switch (transop) { + case LST_TRANS_SESNEW: + case LST_TRANS_SESEND: + rc = lstcon_sesrpc_prep(nd, transop, &rpc); + break; + case LST_TRANS_SESQRY: + case LST_TRANS_SESPING: + rc = lstcon_dbgrpc_prep(nd, &rpc); + break; + case LST_TRANS_TSBCLIADD: + case LST_TRANS_TSBSRVADD: + rc = lstcon_testrpc_prep(nd, transop, + (lstcon_test_t *)arg, &rpc); + break; + case LST_TRANS_TSBRUN: + case LST_TRANS_TSBSTOP: + case LST_TRANS_TSBCLIQRY: + case LST_TRANS_TSBSRVQRY: + rc = lstcon_batrpc_prep(nd, transop, + (lstcon_tsb_hdr_t *)arg, &rpc); + break; + case LST_TRANS_STATQRY: + rc = lstcon_statrpc_prep(nd, &rpc); + break; + default: + rc = -EINVAL; + break; + } + + if (rc != 0) { + CERROR("Failed to create RPC for transaction %s: %d\n", + lstcon_rpc_trans_name(transop), rc); + break; + } + + lstcon_rpc_trans_addreq(trans, rpc); + } + + if (rc == 0) { + *transpp = trans; + return 0; + } + + lstcon_rpc_trans_destroy(trans); + + return rc; +} + +void +lstcon_rpc_pinger(void *arg) +{ + stt_timer_t *ptimer = (stt_timer_t *)arg; + lstcon_rpc_trans_t *trans; + lstcon_rpc_t *crpc; + srpc_msg_t *rep; + srpc_debug_reqst_t *drq; + lstcon_ndlink_t *ndl; + lstcon_node_t *nd; + time_t intv; + int count = 0; + int rc; + + /* RPC pinger is a special case of transaction, + * it's called by timer at 8 seconds interval. + */ + mutex_down(&console_session.ses_mutex); + + if (console_session.ses_shutdown || console_session.ses_expired) { + mutex_up(&console_session.ses_mutex); + return; + } + + if (!console_session.ses_expired && + cfs_time_current_sec() - console_session.ses_laststamp > + console_session.ses_timeout) + console_session.ses_expired = 1; + + trans = console_session.ses_ping; + + LASSERT (trans != NULL); + + list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link) { + nd = ndl->ndl_node; + + if (console_session.ses_expired) { + /* idle console, end session on all nodes */ + if (nd->nd_state != LST_NODE_ACTIVE) + continue; + + rc = lstcon_sesrpc_prep(nd, LST_TRANS_SESEND, &crpc); + if (rc != 0) { + CERROR("Out of memory\n"); + break; + } + + lstcon_rpc_trans_addreq(trans, crpc); + lstcon_rpc_post(crpc); + + continue; + } + + crpc = &nd->nd_ping; + + if (crpc->crp_rpc != NULL) { + LASSERT (crpc->crp_trans == trans); + LASSERT (!list_empty(&crpc->crp_link)); + + spin_lock(&crpc->crp_rpc->crpc_lock); + + LASSERT (crpc->crp_posted); + + if (!crpc->crp_finished) { + /* in flight */ + spin_unlock(&crpc->crp_rpc->crpc_lock); + continue; + } + + spin_unlock(&crpc->crp_rpc->crpc_lock); + + lstcon_rpc_get_reply(crpc, &rep); + + list_del_init(&crpc->crp_link); + + lstcon_rpc_put(crpc); + } + + if (nd->nd_state != LST_NODE_ACTIVE) + continue; + + intv = cfs_duration_sec(cfs_time_sub(cfs_time_current(), + nd->nd_stamp)); + if (intv < nd->nd_timeout / 2) + continue; + + rc = lstcon_rpc_init(nd, SRPC_SERVICE_DEBUG, 0, 0, crpc); + if (rc != 0) { + CERROR("Out of memory\n"); + break; + } + + drq = &crpc->crp_rpc->crpc_reqstmsg.msg_body.dbg_reqst; + + drq->dbg_sid = console_session.ses_id; + drq->dbg_flags = 0; + + lstcon_rpc_trans_addreq(trans, crpc); + lstcon_rpc_post(crpc); + + count ++; + } + + if (console_session.ses_expired) { + mutex_up(&console_session.ses_mutex); + return; + } + + CDEBUG(D_NET, "Ping %d nodes in session\n", count); + + ptimer->stt_expires = cfs_time_current_sec() + LST_PING_INTERVAL; + stt_add_timer(ptimer); + + mutex_up(&console_session.ses_mutex); +} + +int +lstcon_rpc_pinger_start(void) +{ + stt_timer_t *ptimer; + int rc; + + LASSERT (console_session.ses_rpc_pending == 0); + LASSERT (list_empty(&console_session.ses_rpc_list)); + LASSERT (list_empty(&console_session.ses_rpc_freelist)); + + rc = lstcon_rpc_trans_prep(NULL, LST_TRANS_SESPING, + &console_session.ses_ping); + if (rc != 0) { + CERROR("Failed to create console pinger\n"); + return rc; + } + + ptimer = &console_session.ses_ping_timer; + ptimer->stt_expires = cfs_time_current_sec() + LST_PING_INTERVAL; + + stt_add_timer(ptimer); + + return 0; +} + +void +lstcon_rpc_pinger_stop(void) +{ + LASSERT (console_session.ses_shutdown); + + stt_del_timer(&console_session.ses_ping_timer); + + lstcon_rpc_trans_abort(console_session.ses_ping, -ESHUTDOWN); + lstcon_rpc_trans_stat(console_session.ses_ping, lstcon_trans_stat()); + lstcon_rpc_trans_destroy(console_session.ses_ping); + + memset(lstcon_trans_stat(), 0, sizeof(lstcon_trans_stat_t)); + + console_session.ses_ping = NULL; +} + +void +lstcon_rpc_cleanup_wait(void) +{ + lstcon_rpc_trans_t *trans; + lstcon_rpc_t *crpc; + struct list_head *pacer; + struct list_head zlist; + + LASSERT (console_session.ses_shutdown); + + while (!list_empty(&console_session.ses_trans_list)) { + list_for_each(pacer, &console_session.ses_trans_list) { + trans = list_entry(pacer, lstcon_rpc_trans_t, tas_link); + cfs_waitq_signal(&trans->tas_waitq); + + CDEBUG(D_NET, "Session closed, wakeup transaction %s\n", + lstcon_rpc_trans_name(trans->tas_opc)); + } + + mutex_up(&console_session.ses_mutex); + + CWARN("Session is shutting down, close all transactions\n"); + cfs_pause(cfs_time_seconds(1)); + + mutex_down(&console_session.ses_mutex); + } + + spin_lock(&console_session.ses_rpc_lock); + + lst_wait_until(list_empty(&console_session.ses_rpc_list), + console_session.ses_rpc_lock, + "Network is not accessable or target is down, " + "waiting for %d console rpcs to die\n", + console_session.ses_rpc_pending); + + list_add(&zlist, &console_session.ses_rpc_freelist); + list_del_init(&console_session.ses_rpc_freelist); + + spin_unlock(&console_session.ses_rpc_lock); + + LASSERT (console_session.ses_rpc_pending == 0); + + while (!list_empty(&zlist)) { + crpc = list_entry(zlist.next, lstcon_rpc_t, crp_link); + + list_del(&crpc->crp_link); + LIBCFS_FREE(crpc, sizeof(lstcon_rpc_t)); + } +} + +int +lstcon_rpc_module_init(void) +{ + CFS_INIT_LIST_HEAD(&console_session.ses_ping_timer.stt_list); + console_session.ses_ping_timer.stt_func = lstcon_rpc_pinger; + console_session.ses_ping_timer.stt_data = &console_session.ses_ping_timer; + + console_session.ses_ping = NULL; + console_session.ses_rpc_pending = 0; + spin_lock_init(&console_session.ses_rpc_lock); + CFS_INIT_LIST_HEAD(&console_session.ses_rpc_list); + CFS_INIT_LIST_HEAD(&console_session.ses_rpc_freelist); + + return 0; +} + +void +lstcon_rpc_module_fini(void) +{ + LASSERT (console_session.ses_rpc_pending == 0); + LASSERT (list_empty(&console_session.ses_rpc_list)); + LASSERT (list_empty(&console_session.ses_rpc_freelist)); +} + +#endif diff --git a/lnet/selftest/conrpc.h b/lnet/selftest/conrpc.h new file mode 100644 index 0000000..ba6a21b --- /dev/null +++ b/lnet/selftest/conrpc.h @@ -0,0 +1,103 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Author: Liang Zhen + * + * This file is part of Lustre, http://www.lustre.org + * + * Console rpc + */ + +#ifndef __LST_CONRPC_H__ +#define __LST_CONRPC_H__ + +#ifdef __KERNEL__ +#include +#include +#include +#include +#include "rpc.h" +#include "selftest.h" + +/* Console rpc and rpc transaction */ +#define LST_TRANS_TIMEOUT 30 +#define LST_TRANS_MIN_TIMEOUT 3 +#define LST_PING_INTERVAL 8 + +struct lstcon_rpc_trans; +struct lstcon_tsb_hdr; +struct lstcon_test; +struct lstcon_node; + +typedef struct lstcon_rpc { + struct list_head crp_link; /* chain on rpc transaction */ + srpc_client_rpc_t *crp_rpc; /* client rpc */ + struct lstcon_node *crp_node; /* destination node */ + struct lstcon_rpc_trans *crp_trans; /* conrpc transaction */ + + int crp_posted:1; /* rpc is posted */ + int crp_finished:1; /* rpc is finished */ + int crp_unpacked:1; /* reply is unpacked */ + int crp_static:1; /* not from RPC buffer */ + int crp_status; /* console rpc errors */ + cfs_time_t crp_stamp; /* replied time stamp */ +} lstcon_rpc_t; + +typedef struct lstcon_rpc_trans { + struct list_head tas_olink; /* link chain on owner list */ + struct list_head tas_link; /* link chain on global list */ + int tas_opc; /* operation code of transaction */ + cfs_waitq_t tas_waitq; /* wait queue head */ + atomic_t tas_remaining; /* # of un-scheduled rpcs */ + struct list_head tas_rpcs_list; /* queued requests */ +} lstcon_rpc_trans_t; + +#define LST_TRANS_PRIVATE 0x1000 + +#define LST_TRANS_SESNEW (LST_TRANS_PRIVATE | 0x01) +#define LST_TRANS_SESEND (LST_TRANS_PRIVATE | 0x02) +#define LST_TRANS_SESQRY 0x03 +#define LST_TRANS_SESPING 0x04 + +#define LST_TRANS_TSBCLIADD (LST_TRANS_PRIVATE | 0x11) +#define LST_TRANS_TSBSRVADD (LST_TRANS_PRIVATE | 0x12) +#define LST_TRANS_TSBRUN (LST_TRANS_PRIVATE | 0x13) +#define LST_TRANS_TSBSTOP (LST_TRANS_PRIVATE | 0x14) +#define LST_TRANS_TSBCLIQRY 0x15 +#define LST_TRANS_TSBSRVQRY 0x16 + +#define LST_TRANS_STATQRY 0x21 + +typedef int (* lstcon_rpc_cond_func_t)(int, struct lstcon_node *, void *); +typedef int (* lstcon_rpc_readent_func_t)(int, srpc_msg_t *, lstcon_rpc_ent_t *); + +int lstcon_sesrpc_prep(struct lstcon_node *nd, + int transop, lstcon_rpc_t **crpc); +int lstcon_dbgrpc_prep(struct lstcon_node *nd, lstcon_rpc_t **crpc); +int lstcon_batrpc_prep(struct lstcon_node *nd, + int transop, struct lstcon_tsb_hdr *tsb, lstcon_rpc_t **crpc); +int lstcon_testrpc_prep(struct lstcon_node *nd, + int transop, struct lstcon_test *test, lstcon_rpc_t **crpc); +int lstcon_statrpc_prep(struct lstcon_node *nd, lstcon_rpc_t **crpc); +void lstcon_rpc_put(lstcon_rpc_t *crpc); +int lstcon_rpc_trans_prep(struct list_head *translist, + int transop, lstcon_rpc_trans_t **transpp); +int lstcon_rpc_trans_ndlist(struct list_head *ndlist, struct list_head *translist, + int transop, void *arg, lstcon_rpc_cond_func_t condition, + lstcon_rpc_trans_t **transpp); +void lstcon_rpc_trans_stat(lstcon_rpc_trans_t *trans, lstcon_trans_stat_t *stat); +int lstcon_rpc_trans_interpreter(lstcon_rpc_trans_t *trans, struct list_head *head_up, + lstcon_rpc_readent_func_t readent); +void lstcon_rpc_trans_abort(lstcon_rpc_trans_t *trans, int error); +void lstcon_rpc_trans_destroy(lstcon_rpc_trans_t *trans); +void lstcon_rpc_trans_addreq(lstcon_rpc_trans_t *trans, lstcon_rpc_t *req); +int lstcon_rpc_trans_postwait(lstcon_rpc_trans_t *trans, int timeout); +int lstcon_rpc_pinger_start(void); +void lstcon_rpc_pinger_stop(void); +void lstcon_rpc_cleanup_wait(void); +int lstcon_rpc_module_init(void); +void lstcon_rpc_module_fini(void); + +#endif + +#endif diff --git a/lnet/selftest/console.c b/lnet/selftest/console.c new file mode 100644 index 0000000..f474c0b --- /dev/null +++ b/lnet/selftest/console.c @@ -0,0 +1,1979 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Author: Liang Zhen + * + * This file is part of Lustre, http://www.lustre.org + * + * Infrastructure of LST console + */ +#ifdef __KERNEL__ + +#include +#include +#include "console.h" +#include "conrpc.h" + +#define LST_NODE_STATE_COUNTER(nd, p) \ +do { \ + if ((nd)->nd_state == LST_NODE_ACTIVE) \ + (p)->nle_nactive ++; \ + else if ((nd)->nd_state == LST_NODE_BUSY) \ + (p)->nle_nbusy ++; \ + else if ((nd)->nd_state == LST_NODE_DOWN) \ + (p)->nle_ndown ++; \ + else \ + (p)->nle_nunknown ++; \ + (p)->nle_nnode ++; \ +} while (0) + +lstcon_session_t console_session; + +void +lstcon_node_get(lstcon_node_t *nd) +{ + LASSERT (nd->nd_ref >= 1); + + nd->nd_ref++; +} + +static int +lstcon_node_find(lnet_process_id_t id, lstcon_node_t **ndpp, int create) +{ + lstcon_ndlink_t *ndl; + unsigned int idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE; + + LASSERT (id.nid != LNET_NID_ANY); + + list_for_each_entry(ndl, &console_session.ses_ndl_hash[idx], ndl_hlink) { + if (ndl->ndl_node->nd_id.nid != id.nid || + ndl->ndl_node->nd_id.pid != id.pid) + continue; + + lstcon_node_get(ndl->ndl_node); + *ndpp = ndl->ndl_node; + return 0; + } + + if (!create) + return -ENOENT; + + LIBCFS_ALLOC(*ndpp, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t)); + if (*ndpp == NULL) + return -ENOMEM; + + ndl = (lstcon_ndlink_t *)(*ndpp + 1); + + ndl->ndl_node = *ndpp; + + ndl->ndl_node->nd_ref = 1; + ndl->ndl_node->nd_id = id; + ndl->ndl_node->nd_stamp = cfs_time_current(); + ndl->ndl_node->nd_state = LST_NODE_UNKNOWN; + ndl->ndl_node->nd_timeout = 0; + memset(&ndl->ndl_node->nd_ping, 0, sizeof(lstcon_rpc_t)); + + /* queued in global hash & list, no refcount is taken by + * global hash & list, if caller release his refcount, + * node will be released */ + list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]); + list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list); + + return 0; +} + +void +lstcon_node_put(lstcon_node_t *nd) +{ + lstcon_ndlink_t *ndl; + + LASSERT (nd->nd_ref > 0); + + if (--nd->nd_ref > 0) + return; + + ndl = (lstcon_ndlink_t *)(nd + 1); + + LASSERT (!list_empty(&ndl->ndl_link)); + LASSERT (!list_empty(&ndl->ndl_hlink)); + + /* remove from session */ + list_del(&ndl->ndl_link); + list_del(&ndl->ndl_hlink); + + LIBCFS_FREE(nd, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t)); +} + +static int +lstcon_ndlink_find(struct list_head *hash, + lnet_process_id_t id, lstcon_ndlink_t **ndlpp, int create) +{ + unsigned int idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE; + lstcon_ndlink_t *ndl; + lstcon_node_t *nd; + int rc; + + if (id.nid == LNET_NID_ANY) + return -EINVAL; + + /* search in hash */ + list_for_each_entry(ndl, &hash[idx], ndl_hlink) { + if (ndl->ndl_node->nd_id.nid != id.nid || + ndl->ndl_node->nd_id.pid != id.pid) + continue; + + *ndlpp = ndl; + return 0; + } + + if (create == 0) + return -ENOENT; + + /* find or create in session hash */ + rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0); + if (rc != 0) + return rc; + + LIBCFS_ALLOC(ndl, sizeof(lstcon_ndlink_t)); + if (ndl == NULL) { + lstcon_node_put(nd); + return -ENOMEM; + } + + *ndlpp = ndl; + + ndl->ndl_node = nd; + CFS_INIT_LIST_HEAD(&ndl->ndl_link); + list_add_tail(&ndl->ndl_hlink, &hash[idx]); + + return 0; +} + +static void +lstcon_ndlink_release(lstcon_ndlink_t *ndl) +{ + LASSERT (list_empty(&ndl->ndl_link)); + LASSERT (!list_empty(&ndl->ndl_hlink)); + + list_del(&ndl->ndl_hlink); /* delete from hash */ + lstcon_node_put(ndl->ndl_node); + + LIBCFS_FREE(ndl, sizeof(*ndl)); +} + +static int +lstcon_group_alloc(char *name, lstcon_group_t **grpp) +{ + lstcon_group_t *grp; + int i; + + LIBCFS_ALLOC(grp, offsetof(lstcon_group_t, + grp_ndl_hash[LST_NODE_HASHSIZE])); + if (grp == NULL) + return -ENOMEM; + + memset(grp, 0, offsetof(lstcon_group_t, + grp_ndl_hash[LST_NODE_HASHSIZE])); + + grp->grp_ref = 1; + if (name != NULL) + strcpy(grp->grp_name, name); + + CFS_INIT_LIST_HEAD(&grp->grp_link); + CFS_INIT_LIST_HEAD(&grp->grp_ndl_list); + CFS_INIT_LIST_HEAD(&grp->grp_trans_list); + + for (i = 0; i < LST_NODE_HASHSIZE; i++) + CFS_INIT_LIST_HEAD(&grp->grp_ndl_hash[i]); + + *grpp = grp; + + return 0; +} + +static void +lstcon_group_addref(lstcon_group_t *grp) +{ + grp->grp_ref ++; +} + +static void lstcon_group_ndlink_release(lstcon_group_t *, lstcon_ndlink_t *); + +static void +lstcon_group_drain(lstcon_group_t *grp, int keep) +{ + lstcon_ndlink_t *ndl; + lstcon_ndlink_t *tmp; + + list_for_each_entry_safe(ndl, tmp, &grp->grp_ndl_list, ndl_link) { + if ((ndl->ndl_node->nd_state & keep) == 0) + lstcon_group_ndlink_release(grp, ndl); + } +} + +static void +lstcon_group_decref(lstcon_group_t *grp) +{ + int i; + + if (--grp->grp_ref > 0) + return; + + if (!list_empty(&grp->grp_link)) + list_del(&grp->grp_link); + + lstcon_group_drain(grp, 0); + + for (i = 0; i < LST_NODE_HASHSIZE; i++) { + LASSERT (list_empty(&grp->grp_ndl_hash[i])); + } + + LIBCFS_FREE(grp, offsetof(lstcon_group_t, + grp_ndl_hash[LST_NODE_HASHSIZE])); +} + +static int +lstcon_group_find(char *name, lstcon_group_t **grpp) +{ + lstcon_group_t *grp; + + list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) { + if (strncmp(grp->grp_name, name, LST_NAME_SIZE) != 0) + continue; + + lstcon_group_addref(grp); /* +1 ref for caller */ + *grpp = grp; + return 0; + } + + return -ENOENT; +} + +static void +lstcon_group_put(lstcon_group_t *grp) +{ + lstcon_group_decref(grp); +} + +static int +lstcon_group_ndlink_find(lstcon_group_t *grp, lnet_process_id_t id, + lstcon_ndlink_t **ndlpp, int create) +{ + int rc; + + rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create); + if (rc != 0) + return rc; + + if (!list_empty(&(*ndlpp)->ndl_link)) + return 0; + + list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list); + grp->grp_nnode ++; + + return 0; +} + +static void +lstcon_group_ndlink_release(lstcon_group_t *grp, lstcon_ndlink_t *ndl) +{ + list_del_init(&ndl->ndl_link); + lstcon_ndlink_release(ndl); + grp->grp_nnode --; +} + +static void +lstcon_group_ndlink_move(lstcon_group_t *old, + lstcon_group_t *new, lstcon_ndlink_t *ndl) +{ + unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) % + LST_NODE_HASHSIZE; + + list_del(&ndl->ndl_hlink); + list_del(&ndl->ndl_link); + old->grp_nnode --; + + list_add_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]); + list_add_tail(&ndl->ndl_link, &new->grp_ndl_list); + new->grp_nnode ++; + + return; +} + +static void +lstcon_group_move(lstcon_group_t *old, lstcon_group_t *new) +{ + lstcon_ndlink_t *ndl; + + while (!list_empty(&old->grp_ndl_list)) { + ndl = list_entry(old->grp_ndl_list.next, + lstcon_ndlink_t, ndl_link); + lstcon_group_ndlink_move(old, new, ndl); + } +} + +int +lstcon_sesrpc_condition(int transop, lstcon_node_t *nd, void *arg) +{ + lstcon_group_t *grp = (lstcon_group_t *)arg; + + switch (transop) { + case LST_TRANS_SESNEW: + if (nd->nd_state == LST_NODE_ACTIVE) + return 0; + break; + + case LST_TRANS_SESEND: + if (nd->nd_state != LST_NODE_ACTIVE) + return 0; + + if (grp != NULL && nd->nd_ref > 1) + return 0; + break; + + case LST_TRANS_SESQRY: + break; + + default: + LBUG(); + } + + return 1; +} + +int +lstcon_sesrpc_readent(int transop, srpc_msg_t *msg, + lstcon_rpc_ent_t *ent_up) +{ + srpc_debug_reply_t *rep; + + switch (transop) { + case LST_TRANS_SESNEW: + case LST_TRANS_SESEND: + return 0; + + case LST_TRANS_SESQRY: + rep = &msg->msg_body.dbg_reply; + + if (copy_to_user(&ent_up->rpe_priv[0], + &rep->dbg_timeout, sizeof(int)) || + copy_to_user(&ent_up->rpe_payload[0], + &rep->dbg_name, LST_NAME_SIZE)) + return -EFAULT; + + return 0; + + default: + LBUG(); + } + + return 0; +} + +static int +lstcon_group_nodes_add(lstcon_group_t *grp, int count, + lnet_process_id_t *ids_up, struct list_head *result_up) +{ + lstcon_rpc_trans_t *trans; + lstcon_ndlink_t *ndl; + lstcon_group_t *tmp; + lnet_process_id_t id; + int i; + int rc; + + rc = lstcon_group_alloc(NULL, &tmp); + if (rc != 0) { + CERROR("Out of memory\n"); + return -ENOMEM; + } + + for (i = 0 ; i < count; i++) { + if (copy_from_user(&id, &ids_up[i], sizeof(id))) { + rc = -EFAULT; + break; + } + + /* skip if it's in this group already */ + rc = lstcon_group_ndlink_find(grp, id, &ndl, 0); + if (rc == 0) + continue; + + /* add to tmp group */ + rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1); + if (rc != 0) { + CERROR("Can't create ndlink, out of memory\n"); + break; + } + } + + if (rc != 0) { + lstcon_group_put(tmp); + return rc; + } + + rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list, + &tmp->grp_trans_list, LST_TRANS_SESNEW, + tmp, lstcon_sesrpc_condition, &trans); + if (rc != 0) { + CERROR("Can't create transaction: %d\n", rc); + lstcon_group_put(tmp); + return rc; + } + + /* post all RPCs */ + lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT); + + rc = lstcon_rpc_trans_interpreter(trans, result_up, + lstcon_sesrpc_readent); + /* destroy all RPGs */ + lstcon_rpc_trans_destroy(trans); + + lstcon_group_move(tmp, grp); + lstcon_group_put(tmp); + + return rc; +} + +static int +lstcon_group_nodes_remove(lstcon_group_t *grp, + int count, lnet_process_id_t *ids_up, + struct list_head *result_up) +{ + lstcon_rpc_trans_t *trans; + lstcon_ndlink_t *ndl; + lstcon_group_t *tmp; + lnet_process_id_t id; + int rc; + int i; + + /* End session and remove node from the group */ + + rc = lstcon_group_alloc(NULL, &tmp); + if (rc != 0) { + CERROR("Out of memory\n"); + return -ENOMEM; + } + + for (i = 0; i < count; i++) { + if (copy_from_user(&id, &ids_up[i], sizeof(id))) { + rc = -EFAULT; + goto error; + } + + /* move node to tmp group */ + if (lstcon_group_ndlink_find(grp, id, &ndl, 0) == 0) + lstcon_group_ndlink_move(grp, tmp, ndl); + } + + rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list, + &tmp->grp_trans_list, LST_TRANS_SESEND, + tmp, lstcon_sesrpc_condition, &trans); + if (rc != 0) { + CERROR("Can't create transaction: %d\n", rc); + goto error; + } + + lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT); + + rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL); + + lstcon_rpc_trans_destroy(trans); + /* release nodes anyway, because we can't rollback status */ + lstcon_group_put(tmp); + + return rc; +error: + lstcon_group_move(tmp, grp); + lstcon_group_put(tmp); + + return rc; +} + +int +lstcon_group_add(char *name) +{ + lstcon_group_t *grp; + int rc; + + rc = (lstcon_group_find(name, &grp) == 0)? -EEXIST: 0; + if (rc != 0) { + /* find a group with same name */ + lstcon_group_put(grp); + return rc; + } + + rc = lstcon_group_alloc(name, &grp); + if (rc != 0) { + CERROR("Can't allocate descriptor for group %s\n", name); + return -ENOMEM; + } + + list_add_tail(&grp->grp_link, &console_session.ses_grp_list); + + return rc; +} + +int +lstcon_nodes_add(char *name, int count, + lnet_process_id_t *ids_up, struct list_head *result_up) +{ + lstcon_group_t *grp; + int rc; + + LASSERT (count > 0); + LASSERT (ids_up != NULL); + + rc = lstcon_group_find(name, &grp); + if (rc != 0) { + CDEBUG(D_NET, "Can't find group %s\n", name); + return rc; + } + + if (grp->grp_ref > 2) { + /* referred by other threads or test */ + CDEBUG(D_NET, "Group %s is busy\n", name); + lstcon_group_put(grp); + + return -EBUSY; + } + + rc = lstcon_group_nodes_add(grp, count, ids_up, result_up); + + lstcon_group_put(grp); + + return rc; +} + +int +lstcon_group_del(char *name) +{ + lstcon_rpc_trans_t *trans; + lstcon_group_t *grp; + int rc; + + rc = lstcon_group_find(name, &grp); + if (rc != 0) { + CDEBUG(D_NET, "Can't find group: %s\n", name); + return rc; + } + + if (grp->grp_ref > 2) { + /* referred by others threads or test */ + CDEBUG(D_NET, "Group %s is busy\n", name); + lstcon_group_put(grp); + return -EBUSY; + } + + rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list, + &grp->grp_trans_list, LST_TRANS_SESEND, + grp, lstcon_sesrpc_condition, &trans); + if (rc != 0) { + CERROR("Can't create transaction: %d\n", rc); + lstcon_group_put(grp); + return rc; + } + + lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT); + + lstcon_rpc_trans_destroy(trans); + + lstcon_group_put(grp); + /* -ref for session, it's destroyed, + * status can't be rolled back, destroy group anway */ + lstcon_group_put(grp); + + return rc; +} + +int +lstcon_group_clean(char *name, int args) +{ + lstcon_group_t *grp = NULL; + int rc; + + rc = lstcon_group_find(name, &grp); + if (rc != 0) { + CDEBUG(D_NET, "Can't find group %s\n", name); + return rc; + } + + if (grp->grp_ref > 2) { + /* referred by test */ + CDEBUG(D_NET, "Group %s is busy\n", name); + lstcon_group_put(grp); + return -EBUSY; + } + + args = (LST_NODE_ACTIVE | LST_NODE_BUSY | + LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args; + + lstcon_group_drain(grp, args); + + lstcon_group_put(grp); + /* release empty group */ + if (list_empty(&grp->grp_ndl_list)) + lstcon_group_put(grp); + + return 0; +} + +int +lstcon_nodes_remove(char *name, int count, + lnet_process_id_t *ids_up, struct list_head *result_up) +{ + lstcon_group_t *grp = NULL; + int rc; + + rc = lstcon_group_find(name, &grp); + if (rc != 0) { + CDEBUG(D_NET, "Can't find group: %s\n", name); + return rc; + } + + if (grp->grp_ref > 2) { + /* referred by test */ + CDEBUG(D_NET, "Group %s is busy\n", name); + lstcon_group_put(grp); + return -EBUSY; + } + + rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up); + + lstcon_group_put(grp); + /* release empty group */ + if (list_empty(&grp->grp_ndl_list)) + lstcon_group_put(grp); + + return rc; +} + +int +lstcon_group_refresh(char *name, struct list_head *result_up) +{ + lstcon_rpc_trans_t *trans; + lstcon_group_t *grp; + int rc; + + rc = lstcon_group_find(name, &grp); + if (rc != 0) { + CDEBUG(D_NET, "Can't find group: %s\n", name); + return rc; + } + + if (grp->grp_ref > 2) { + /* referred by test */ + CDEBUG(D_NET, "Group %s is busy\n", name); + lstcon_group_put(grp); + return -EBUSY; + } + + /* re-invite all inactive nodes int the group */ + rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list, + &grp->grp_trans_list, LST_TRANS_SESNEW, + grp, lstcon_sesrpc_condition, &trans); + if (rc != 0) { + /* local error, return */ + CDEBUG(D_NET, "Can't create transaction: %d\n", rc); + lstcon_group_put(grp); + return rc; + } + + lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT); + + rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL); + + lstcon_rpc_trans_destroy(trans); + /* -ref for me */ + lstcon_group_put(grp); + + return rc; +} + +int +lstcon_group_list(int index, int len, char *name_up) +{ + lstcon_group_t *grp; + + LASSERT (index >= 0); + LASSERT (name_up != NULL); + + list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) { + if (index-- == 0) { + return copy_to_user(name_up, grp->grp_name, len) ? + -EFAULT : 0; + } + } + + return -ENOENT; +} + +static int +lstcon_nodes_getent(struct list_head *head, int *index_p, + int *count_p, lstcon_node_ent_t *dents_up) +{ + lstcon_ndlink_t *ndl; + lstcon_node_t *nd; + int count = 0; + int index = 0; + + LASSERT (index_p != NULL && count_p != NULL); + LASSERT (dents_up != NULL); + LASSERT (*index_p >= 0); + LASSERT (*count_p > 0); + + list_for_each_entry(ndl, head, ndl_link) { + if (index++ < *index_p) + continue; + + if (count >= *count_p) + break; + + nd = ndl->ndl_node; + if (copy_to_user(&dents_up[count].nde_id, + &nd->nd_id, sizeof(nd->nd_id)) || + copy_to_user(&dents_up[count].nde_state, + &nd->nd_state, sizeof(nd->nd_state))) + return -EFAULT; + + count ++; + } + + if (index <= *index_p) + return -ENOENT; + + *count_p = count; + *index_p = index; + + return 0; +} + +int +lstcon_group_info(char *name, lstcon_ndlist_ent_t *gents_p, + int *index_p, int *count_p, lstcon_node_ent_t *dents_up) +{ + lstcon_ndlist_ent_t *gentp; + lstcon_group_t *grp; + lstcon_ndlink_t *ndl; + int rc; + + rc = lstcon_group_find(name, &grp); + if (rc != 0) { + CDEBUG(D_NET, "Can't find group %s\n", name); + return rc; + } + + if (dents_up != 0) { + /* verbose query */ + rc = lstcon_nodes_getent(&grp->grp_ndl_list, + index_p, count_p, dents_up); + lstcon_group_put(grp); + + return rc; + } + + /* non-verbose query */ + LIBCFS_ALLOC(gentp, sizeof(lstcon_ndlist_ent_t)); + if (gentp == NULL) { + CERROR("Can't allocate ndlist_ent\n"); + lstcon_group_put(grp); + + return -ENOMEM; + } + + memset(gentp, 0, sizeof(lstcon_ndlist_ent_t)); + + list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link) + LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp); + + rc = copy_to_user(gents_p, gentp, + sizeof(lstcon_ndlist_ent_t)) ? -EFAULT: 0; + + LIBCFS_FREE(gentp, sizeof(lstcon_ndlist_ent_t)); + + lstcon_group_put(grp); + + return 0; +} + +int +lstcon_batch_find(char *name, lstcon_batch_t **batpp) +{ + lstcon_batch_t *bat; + + list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) { + if (strncmp(bat->bat_name, name, LST_NAME_SIZE) == 0) { + *batpp = bat; + return 0; + } + } + + return -ENOENT; +} + +int +lstcon_batch_add(char *name) +{ + lstcon_batch_t *bat; + int i; + int rc; + + rc = (lstcon_batch_find(name, &bat) == 0)? -EEXIST: 0; + if (rc != 0) { + CDEBUG(D_NET, "Batch %s already exists\n", name); + return rc; + } + + LIBCFS_ALLOC(bat, sizeof(lstcon_batch_t)); + if (bat == NULL) { + CERROR("Can't allocate descriptor for batch %s\n", name); + return -ENOMEM; + } + + LIBCFS_ALLOC(bat->bat_cli_hash, + sizeof(struct list_head) * LST_NODE_HASHSIZE); + if (bat->bat_cli_hash == NULL) { + CERROR("Can't allocate hash for batch %s\n", name); + LIBCFS_FREE(bat, sizeof(lstcon_batch_t)); + + return -ENOMEM; + } + + LIBCFS_ALLOC(bat->bat_srv_hash, + sizeof(struct list_head) * LST_NODE_HASHSIZE); + if (bat->bat_srv_hash == NULL) { + CERROR("Can't allocate hash for batch %s\n", name); + LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE); + LIBCFS_FREE(bat, sizeof(lstcon_batch_t)); + + return -ENOMEM; + } + + strcpy(bat->bat_name, name); + bat->bat_hdr.tsb_index = 0; + bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie; + + bat->bat_ntest = 0; + bat->bat_state = LST_BATCH_IDLE; + + CFS_INIT_LIST_HEAD(&bat->bat_cli_list); + CFS_INIT_LIST_HEAD(&bat->bat_srv_list); + CFS_INIT_LIST_HEAD(&bat->bat_test_list); + CFS_INIT_LIST_HEAD(&bat->bat_trans_list); + + for (i = 0; i < LST_NODE_HASHSIZE; i++) { + CFS_INIT_LIST_HEAD(&bat->bat_cli_hash[i]); + CFS_INIT_LIST_HEAD(&bat->bat_srv_hash[i]); + } + + list_add_tail(&bat->bat_link, &console_session.ses_bat_list); + + return rc; +} + +int +lstcon_batch_list(int index, int len, char *name_up) +{ + lstcon_batch_t *bat; + + LASSERT (name_up != NULL); + LASSERT (index >= 0); + + list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) { + if (index-- == 0) { + return copy_to_user(name_up,bat->bat_name, len) ? + -EFAULT: 0; + } + } + + return -ENOENT; +} + +int +lstcon_batch_info(char *name, lstcon_test_batch_ent_t *ent_up, int server, + int testidx, int *index_p, int *ndent_p, + lstcon_node_ent_t *dents_up) +{ + lstcon_test_batch_ent_t *entp; + struct list_head *clilst; + struct list_head *srvlst; + lstcon_test_t *test = NULL; + lstcon_batch_t *bat; + lstcon_ndlink_t *ndl; + int rc; + + rc = lstcon_batch_find(name, &bat); + if (rc != 0) { + CDEBUG(D_NET, "Can't find batch %s\n", name); + return -ENOENT; + } + + if (testidx > 0) { + /* query test, test index start from 1 */ + list_for_each_entry(test, &bat->bat_test_list, tes_link) { + if (testidx-- == 1) + break; + } + + if (testidx > 0) { + CDEBUG(D_NET, "Can't find specified test in batch\n"); + return -ENOENT; + } + } + + clilst = (test == NULL) ? &bat->bat_cli_list : + &test->tes_src_grp->grp_ndl_list; + srvlst = (test == NULL) ? &bat->bat_srv_list : + &test->tes_dst_grp->grp_ndl_list; + + if (dents_up != NULL) { + rc = lstcon_nodes_getent((server ? srvlst: clilst), + index_p, ndent_p, dents_up); + return rc; + } + + /* non-verbose query */ + LIBCFS_ALLOC(entp, sizeof(lstcon_test_batch_ent_t)); + if (entp == NULL) + return -ENOMEM; + + memset(entp, 0, sizeof(lstcon_test_batch_ent_t)); + + if (test == NULL) { + entp->u.tbe_batch.bae_ntest = bat->bat_ntest; + entp->u.tbe_batch.bae_state = bat->bat_state; + + } else { + + entp->u.tbe_test.tse_type = test->tes_type; + entp->u.tbe_test.tse_loop = test->tes_loop; + entp->u.tbe_test.tse_concur = test->tes_concur; + } + + list_for_each_entry(ndl, clilst, ndl_link) + LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle); + + list_for_each_entry(ndl, srvlst, ndl_link) + LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle); + + rc = copy_to_user(ent_up, entp, + sizeof(lstcon_test_batch_ent_t)) ? -EFAULT : 0; + + LIBCFS_FREE(entp, sizeof(lstcon_test_batch_ent_t)); + + return rc; +} + +int +lstcon_batrpc_condition(int transop, lstcon_node_t *nd, void *arg) +{ + switch (transop) { + case LST_TRANS_TSBRUN: + if (nd->nd_state != LST_NODE_ACTIVE) + return -ENETDOWN; + break; + + case LST_TRANS_TSBSTOP: + if (nd->nd_state != LST_NODE_ACTIVE) + return 0; + break; + + case LST_TRANS_TSBCLIQRY: + case LST_TRANS_TSBSRVQRY: + break; + } + + return 1; +} + +static int +lstcon_batch_op(lstcon_batch_t *bat, int transop, struct list_head *result_up) +{ + lstcon_rpc_trans_t *trans; + int rc; + + rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list, + &bat->bat_trans_list, transop, + bat, lstcon_batrpc_condition, &trans); + if (rc != 0) { + CERROR("Can't create transaction: %d\n", rc); + return rc; + } + + lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT); + + rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL); + + lstcon_rpc_trans_destroy(trans); + + return rc; +} + +int +lstcon_batch_run(char *name, int timeout, struct list_head *result_up) +{ + lstcon_batch_t *bat; + int rc; + + if (lstcon_batch_find(name, &bat) != 0) { + CDEBUG(D_NET, "Can't find batch %s\n", name); + return -ENOENT; + } + + bat->bat_arg = timeout; + + rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up); + + /* mark batch as running if it's started in any node */ + if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0) != 0) + bat->bat_state = LST_BATCH_RUNNING; + + return rc; +} + +int +lstcon_batch_stop(char *name, int force, struct list_head *result_up) +{ + lstcon_batch_t *bat; + int rc; + + if (lstcon_batch_find(name, &bat) != 0) { + CDEBUG(D_NET, "Can't find batch %s\n", name); + return -ENOENT; + } + + bat->bat_arg = force; + + rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up); + + /* mark batch as stopped if all RPCs finished */ + if (lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0) == 0) + bat->bat_state = LST_BATCH_IDLE; + + return rc; +} + +static void +lstcon_batch_destroy(lstcon_batch_t *bat) +{ + lstcon_ndlink_t *ndl; + lstcon_test_t *test; + int i; + + list_del(&bat->bat_link); + + while (!list_empty(&bat->bat_test_list)) { + test = list_entry(bat->bat_test_list.next, + lstcon_test_t, tes_link); + LASSERT (list_empty(&test->tes_trans_list)); + + list_del(&test->tes_link); + + lstcon_group_put(test->tes_src_grp); + lstcon_group_put(test->tes_dst_grp); + + LIBCFS_FREE(test, offsetof(lstcon_test_t, + tes_param[test->tes_paramlen])); + } + + LASSERT (list_empty(&bat->bat_trans_list)); + + while (!list_empty(&bat->bat_cli_list)) { + ndl = list_entry(bat->bat_cli_list.next, + lstcon_ndlink_t, ndl_link); + list_del_init(&ndl->ndl_link); + + lstcon_ndlink_release(ndl); + } + + while (!list_empty(&bat->bat_srv_list)) { + ndl = list_entry(bat->bat_srv_list.next, + lstcon_ndlink_t, ndl_link); + list_del_init(&ndl->ndl_link); + + lstcon_ndlink_release(ndl); + } + + for (i = 0; i < LST_NODE_HASHSIZE; i++) { + LASSERT (list_empty(&bat->bat_cli_hash[i])); + LASSERT (list_empty(&bat->bat_srv_hash[i])); + } + + LIBCFS_FREE(bat->bat_cli_hash, + sizeof(struct list_head) * LST_NODE_HASHSIZE); + LIBCFS_FREE(bat->bat_srv_hash, + sizeof(struct list_head) * LST_NODE_HASHSIZE); + LIBCFS_FREE(bat, sizeof(lstcon_batch_t)); +} + +int +lstcon_testrpc_condition(int transop, lstcon_node_t *nd, void *arg) +{ + lstcon_test_t *test; + lstcon_batch_t *batch; + lstcon_ndlink_t *ndl; + struct list_head *hash; + struct list_head *head; + + test = (lstcon_test_t *)arg; + LASSERT (test != NULL); + + batch = test->tes_batch; + LASSERT (batch != NULL); + + if (test->tes_oneside && + transop == LST_TRANS_TSBSRVADD) + return 0; + + if (nd->nd_state != LST_NODE_ACTIVE) + return -ENETDOWN; + + if (transop == LST_TRANS_TSBCLIADD) { + hash = batch->bat_cli_hash; + head = &batch->bat_cli_list; + + } else { + LASSERT (transop == LST_TRANS_TSBSRVADD); + + hash = batch->bat_srv_hash; + head = &batch->bat_srv_list; + } + + LASSERT (nd->nd_id.nid != LNET_NID_ANY); + + if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1) != 0) + return -ENOMEM; + + if (list_empty(&ndl->ndl_link)) + list_add_tail(&ndl->ndl_link, head); + + return 1; +} + +static int +lstcon_test_nodes_add(lstcon_test_t *test, struct list_head *result_up) +{ + lstcon_rpc_trans_t *trans; + lstcon_group_t *grp; + int transop; + int rc; + + LASSERT (test->tes_src_grp != NULL); + LASSERT (test->tes_dst_grp != NULL); + + transop = LST_TRANS_TSBSRVADD; + grp = test->tes_dst_grp; +again: + rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list, + &test->tes_trans_list, transop, + test, lstcon_testrpc_condition, &trans); + if (rc != 0) { + CERROR("Can't create transaction: %d\n", rc); + return rc; + } + + lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT); + + if (lstcon_trans_stat()->trs_rpc_errno != 0 || + lstcon_trans_stat()->trs_fwk_errno != 0) { + lstcon_rpc_trans_interpreter(trans, result_up, NULL); + + lstcon_rpc_trans_destroy(trans); + /* return if any error */ + CDEBUG(D_NET, "Failed to add test %s, " + "RPC error %d, framework error %d\n", + transop == LST_TRANS_TSBCLIADD ? "client" : "server", + lstcon_trans_stat()->trs_rpc_errno, + lstcon_trans_stat()->trs_fwk_errno); + + return rc; + } + + lstcon_rpc_trans_destroy(trans); + + if (transop == LST_TRANS_TSBCLIADD) + return rc; + + transop = LST_TRANS_TSBCLIADD; + grp = test->tes_src_grp; + test->tes_cliidx = 0; + + /* requests to test clients */ + goto again; +} + +int +lstcon_test_add(char *name, int type, int loop, int concur, + int dist, int span, char *src_name, char * dst_name, + void *param, int paramlen, struct list_head *result_up) + +{ + lstcon_group_t *src_grp = NULL; + lstcon_group_t *dst_grp = NULL; + lstcon_test_t *test = NULL; + lstcon_batch_t *batch; + int rc; + + rc = lstcon_batch_find(name, &batch); + if (rc != 0) { + CDEBUG(D_NET, "Can't find batch %s\n", name); + return rc; + } + + if (batch->bat_state != LST_BATCH_IDLE) { + CDEBUG(D_NET, "Can't change running batch %s\n", name); + return rc; + } + + rc = lstcon_group_find(src_name, &src_grp); + if (rc != 0) { + CDEBUG(D_NET, "Can't find group %s\n", src_name); + goto out; + } + + rc = lstcon_group_find(dst_name, &dst_grp); + if (rc != 0) { + CDEBUG(D_NET, "Can't find group %s\n", dst_name); + goto out; + } + + LIBCFS_ALLOC(test, offsetof(lstcon_test_t, tes_param[paramlen])); + if (!test) { + CERROR("Can't allocate test descriptor\n"); + rc = -ENOMEM; + + goto out; + } + + memset(test, 0, offsetof(lstcon_test_t, tes_param[paramlen])); + test->tes_hdr.tsb_id = batch->bat_hdr.tsb_id; + test->tes_batch = batch; + test->tes_type = type; + test->tes_oneside = 0; /* TODO */ + test->tes_loop = loop; + test->tes_concur = concur; + test->tes_stop_onerr = 1; /* TODO */ + test->tes_span = span; + test->tes_dist = dist; + test->tes_cliidx = 0; /* just used for creating RPC */ + test->tes_src_grp = src_grp; + test->tes_dst_grp = dst_grp; + CFS_INIT_LIST_HEAD(&test->tes_trans_list); + + if (param != NULL) { + test->tes_paramlen = paramlen; + memcpy(&test->tes_param[0], param, paramlen); + } + + rc = lstcon_test_nodes_add(test, result_up); + + if (rc != 0) + goto out; + + if (lstcon_trans_stat()->trs_rpc_errno != 0 || + lstcon_trans_stat()->trs_fwk_errno != 0) + CDEBUG(D_NET, "Failed to add test %d to batch %s", type, name); + + /* add to test list anyway, so user can check what's going on */ + list_add_tail(&test->tes_link, &batch->bat_test_list); + + batch->bat_ntest ++; + test->tes_hdr.tsb_index = batch->bat_ntest; + + /* hold groups so nobody can change them */ + return rc; +out: + if (test != NULL) + LIBCFS_FREE(test, offsetof(lstcon_test_t, tes_param[paramlen])); + + if (dst_grp != NULL) + lstcon_group_put(dst_grp); + + if (src_grp != NULL) + lstcon_group_put(src_grp); + + return rc; +} + +int +lstcon_test_find(lstcon_batch_t *batch, int idx, lstcon_test_t **testpp) +{ + lstcon_test_t *test; + + list_for_each_entry(test, &batch->bat_test_list, tes_link) { + if (idx == test->tes_hdr.tsb_index) { + *testpp = test; + return 0; + } + } + + return -ENOENT; +} + +int +lstcon_tsbrpc_readent(int transop, srpc_msg_t *msg, + lstcon_rpc_ent_t *ent_up) +{ + srpc_batch_reply_t *rep = &msg->msg_body.bat_reply; + + LASSERT (transop == LST_TRANS_TSBCLIQRY || + transop == LST_TRANS_TSBSRVQRY); + + /* positive errno, framework error code */ + if (copy_to_user(&ent_up->rpe_priv[0], + &rep->bar_active, sizeof(rep->bar_active))) + return -EFAULT; + + return 0; +} + +int +lstcon_test_batch_query(char *name, int testidx, int client, + int timeout, struct list_head *result_up) +{ + lstcon_rpc_trans_t *trans; + struct list_head *translist; + struct list_head *ndlist; + lstcon_tsb_hdr_t *hdr; + lstcon_batch_t *batch; + lstcon_test_t *test = NULL; + int transop; + int rc; + + rc = lstcon_batch_find(name, &batch); + if (rc != 0) { + CDEBUG(D_NET, "Can't find batch: %s\n", name); + return rc; + } + + if (testidx == 0) { + translist = &batch->bat_trans_list; + ndlist = &batch->bat_cli_list; + hdr = &batch->bat_hdr; + + } else { + /* query specified test only */ + rc = lstcon_test_find(batch, testidx, &test); + if (rc != 0) { + CDEBUG(D_NET, "Can't find test: %d\n", testidx); + return rc; + } + + translist = &test->tes_trans_list; + ndlist = &test->tes_src_grp->grp_ndl_list; + hdr = &test->tes_hdr; + } + + transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY; + + rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr, + lstcon_batrpc_condition, &trans); + if (rc != 0) { + CERROR("Can't create transaction: %d\n", rc); + return rc; + } + + lstcon_rpc_trans_postwait(trans, timeout); + + if (testidx == 0 && /* query a batch, not a test */ + lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) == 0 && + lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0) == 0) { + /* all RPCs finished, and no active test */ + batch->bat_state = LST_BATCH_IDLE; + } + + rc = lstcon_rpc_trans_interpreter(trans, result_up, + lstcon_tsbrpc_readent); + lstcon_rpc_trans_destroy(trans); + + return rc; +} + +int +lstcon_statrpc_readent(int transop, srpc_msg_t *msg, + lstcon_rpc_ent_t *ent_up) +{ + srpc_stat_reply_t *rep = &msg->msg_body.stat_reply; + sfw_counters_t *sfwk_stat; + srpc_counters_t *srpc_stat; + lnet_counters_t *lnet_stat; + + if (rep->str_status != 0) + return 0; + + sfwk_stat = (sfw_counters_t *)&ent_up->rpe_payload[0]; + srpc_stat = (srpc_counters_t *)((char *)sfwk_stat + sizeof(*sfwk_stat)); + lnet_stat = (lnet_counters_t *)((char *)srpc_stat + sizeof(*srpc_stat)); + + if (copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) || + copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) || + copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat))) + return -EFAULT; + + return 0; +} + +int +lstcon_ndlist_stat(struct list_head *ndlist, + int timeout, struct list_head *result_up) +{ + struct list_head head; + lstcon_rpc_trans_t *trans; + int rc; + + CFS_INIT_LIST_HEAD(&head); + + rc = lstcon_rpc_trans_ndlist(ndlist, &head, + LST_TRANS_STATQRY, NULL, NULL, &trans); + if (rc != 0) { + CERROR("Can't create transaction: %d\n", rc); + return rc; + } + + timeout = (timeout > LST_TRANS_MIN_TIMEOUT) ? timeout : + LST_TRANS_MIN_TIMEOUT; + lstcon_rpc_trans_postwait(trans, timeout); + + rc = lstcon_rpc_trans_interpreter(trans, result_up, + lstcon_statrpc_readent); + lstcon_rpc_trans_destroy(trans); + + return rc; +} + +int +lstcon_group_stat(char *grp_name, int timeout, struct list_head *result_up) +{ + lstcon_group_t *grp; + int rc; + + rc = lstcon_group_find(grp_name, &grp); + if (rc != 0) { + CDEBUG(D_NET, "Can't find group %s\n", grp_name); + return rc; + } + + rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up); + + lstcon_group_put(grp); + + return rc; +} + +int +lstcon_nodes_stat(int count, lnet_process_id_t *ids_up, + int timeout, struct list_head *result_up) +{ + lstcon_ndlink_t *ndl; + lstcon_group_t *tmp; + lnet_process_id_t id; + int i; + int rc; + + rc = lstcon_group_alloc(NULL, &tmp); + if (rc != 0) { + CERROR("Out of memory\n"); + return -ENOMEM; + } + + for (i = 0 ; i < count; i++) { + if (copy_from_user(&id, &ids_up[i], sizeof(id))) { + rc = -EFAULT; + break; + } + + /* add to tmp group */ + rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2); + if (rc != 0) { + CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET, + "Failed to find or create %s: %d\n", + libcfs_id2str(id), rc); + break; + } + } + + if (rc != 0) { + lstcon_group_put(tmp); + return rc; + } + + rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up); + + lstcon_group_put(tmp); + + return rc; +} + +int +lstcon_debug_ndlist(struct list_head *ndlist, + struct list_head *translist, + int timeout, struct list_head *result_up) +{ + lstcon_rpc_trans_t *trans; + int rc; + + rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY, + NULL, lstcon_sesrpc_condition, &trans); + if (rc != 0) { + CERROR("Can't create transaction: %d\n", rc); + return rc; + } + + timeout = (timeout > LST_TRANS_MIN_TIMEOUT) ? timeout : + LST_TRANS_MIN_TIMEOUT; + + lstcon_rpc_trans_postwait(trans, timeout); + + rc = lstcon_rpc_trans_interpreter(trans, result_up, + lstcon_sesrpc_readent); + lstcon_rpc_trans_destroy(trans); + + return rc; +} + +int +lstcon_session_debug(int timeout, struct list_head *result_up) +{ + return lstcon_debug_ndlist(&console_session.ses_ndl_list, + NULL, timeout, result_up); +} + +int +lstcon_batch_debug(int timeout, char *name, + int client, struct list_head *result_up) +{ + lstcon_batch_t *bat; + int rc; + + rc = lstcon_batch_find(name, &bat); + if (rc != 0) + return -ENOENT; + + rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list : + &bat->bat_srv_list, + NULL, timeout, result_up); + + return rc; +} + +int +lstcon_group_debug(int timeout, char *name, + struct list_head *result_up) +{ + lstcon_group_t *grp; + int rc; + + rc = lstcon_group_find(name, &grp); + if (rc != 0) + return -ENOENT; + + rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL, + timeout, result_up); + lstcon_group_put(grp); + + return rc; +} + +int +lstcon_nodes_debug(int timeout, + int count, lnet_process_id_t *ids_up, + struct list_head *result_up) +{ + lnet_process_id_t id; + lstcon_ndlink_t *ndl; + lstcon_group_t *grp; + int i; + int rc; + + rc = lstcon_group_alloc(NULL, &grp); + if (rc != 0) { + CDEBUG(D_NET, "Out of memory\n"); + return rc; + } + + for (i = 0; i < count; i++) { + if (copy_from_user(&id, &ids_up[i], sizeof(id))) { + rc = -EFAULT; + break; + } + + /* node is added to tmp group */ + rc = lstcon_group_ndlink_find(grp, id, &ndl, 1); + if (rc != 0) { + CERROR("Can't create node link\n"); + break; + } + } + + if (rc != 0) { + lstcon_group_put(grp); + return rc; + } + + rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL, + timeout, result_up); + + lstcon_group_put(grp); + + return rc; +} + +int +lstcon_session_match(lst_sid_t sid) +{ + return (console_session.ses_id.ses_nid == sid.ses_nid && + console_session.ses_id.ses_stamp == sid.ses_stamp) ? 1: 0; +} + +static void +lstcon_new_session_id(lst_sid_t *sid) +{ + lnet_process_id_t id; + + LASSERT (console_session.ses_state == LST_SESSION_NONE); + + LNetGetId(1, &id); + sid->ses_nid = id.nid; + sid->ses_stamp = cfs_time_current(); +} + +extern srpc_service_t lstcon_acceptor_service; + +int +lstcon_session_new(char *name, int key, + int timeout,int force, lst_sid_t *sid_up) +{ + int rc = 0; + int i; + + if (console_session.ses_state != LST_SESSION_NONE) { + /* session exists */ + if (!force) { + CERROR("Session %s already exists\n", + console_session.ses_name); + return -EEXIST; + } + + rc = lstcon_session_end(); + + /* lstcon_session_end() only return local error */ + if (rc != 0) + return rc; + } + + for (i = 0; i < LST_GLOBAL_HASHSIZE; i++) { + LASSERT (list_empty(&console_session.ses_ndl_hash[i])); + } + + rc = lstcon_batch_add(LST_DEFAULT_BATCH); + if (rc != 0) + return rc; + + rc = lstcon_rpc_pinger_start(); + if (rc != 0) { + lstcon_batch_t *bat; + + lstcon_batch_find(LST_DEFAULT_BATCH, &bat); + lstcon_batch_destroy(bat); + + return rc; + } + + lstcon_new_session_id(&console_session.ses_id); + + console_session.ses_key = key; + console_session.ses_state = LST_SESSION_ACTIVE; + console_session.ses_force = !!force; + console_session.ses_timeout = (timeout <= 0)? LST_CONSOLE_TIMEOUT: + timeout; + strcpy(console_session.ses_name, name); + + if (copy_to_user(sid_up, &console_session.ses_id, + sizeof(lst_sid_t)) == 0) + return rc; + + lstcon_session_end(); + + return -EFAULT; +} + +int +lstcon_session_info(lst_sid_t *sid_up, int *key_up, + lstcon_ndlist_ent_t *ndinfo_up, char *name_up, int len) +{ + lstcon_ndlist_ent_t *entp; + lstcon_ndlink_t *ndl; + int rc = 0; + + if (console_session.ses_state != LST_SESSION_ACTIVE) + return -ESRCH; + + LIBCFS_ALLOC(entp, sizeof(*entp)); + if (entp == NULL) + return -ENOMEM; + + memset(entp, 0, sizeof(*entp)); + + list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link) + LST_NODE_STATE_COUNTER(ndl->ndl_node, entp); + + if (copy_to_user(sid_up, &console_session.ses_id, sizeof(lst_sid_t)) || + copy_to_user(key_up, &console_session.ses_key, sizeof(int)) || + copy_to_user(ndinfo_up, entp, sizeof(*entp)) || + copy_to_user(name_up, console_session.ses_name, len)) + rc = -EFAULT; + + LIBCFS_FREE(entp, sizeof(*entp)); + + return rc; +} + +int +lstcon_session_end() +{ + lstcon_rpc_trans_t *trans; + lstcon_group_t *grp; + lstcon_batch_t *bat; + int rc = 0; + + LASSERT (console_session.ses_state == LST_SESSION_ACTIVE); + + rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list, + NULL, LST_TRANS_SESEND, NULL, + lstcon_sesrpc_condition, &trans); + if (rc != 0) { + CERROR("Can't create transaction: %d\n", rc); + return rc; + } + + console_session.ses_shutdown = 1; + + lstcon_rpc_pinger_stop(); + + lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT); + + lstcon_rpc_trans_destroy(trans); + /* User can do nothing even rpc failed, so go on */ + + /* waiting for orphan rpcs to die */ + lstcon_rpc_cleanup_wait(); + + console_session.ses_id = LST_INVALID_SID; + console_session.ses_state = LST_SESSION_NONE; + console_session.ses_key = 0; + console_session.ses_force = 0; + + /* destroy all batches */ + while (!list_empty(&console_session.ses_bat_list)) { + bat = list_entry(console_session.ses_bat_list.next, + lstcon_batch_t, bat_link); + + lstcon_batch_destroy(bat); + } + + /* destroy all groups */ + while (!list_empty(&console_session.ses_grp_list)) { + grp = list_entry(console_session.ses_grp_list.next, + lstcon_group_t, grp_link); + LASSERT (grp->grp_ref == 1); + + lstcon_group_put(grp); + } + + /* all nodes should be released */ + LASSERT (list_empty(&console_session.ses_ndl_list)); + + console_session.ses_shutdown = 0; + console_session.ses_expired = 0; + + return rc; +} + +static int +lstcon_acceptor_handle (srpc_server_rpc_t *rpc) +{ + srpc_msg_t *rep = &rpc->srpc_replymsg; + srpc_msg_t *req = &rpc->srpc_reqstbuf->buf_msg; + srpc_join_reqst_t *jreq = &req->msg_body.join_reqst; + srpc_join_reply_t *jrep = &rep->msg_body.join_reply; + lstcon_group_t *grp = NULL; + lstcon_ndlink_t *ndl; + int rc = 0; + + sfw_unpack_message(req); + + mutex_down(&console_session.ses_mutex); + + jrep->join_sid = console_session.ses_id; + + if (console_session.ses_id.ses_nid == LNET_NID_ANY) { + jrep->join_status = ESRCH; + goto out; + } + + if (jreq->join_sid.ses_nid != LNET_NID_ANY && + !lstcon_session_match(jreq->join_sid)) { + jrep->join_status = EBUSY; + goto out; + } + + if (lstcon_group_find(jreq->join_group, &grp) != 0) { + rc = lstcon_group_alloc(jreq->join_group, &grp); + if (rc != 0) { + CERROR("Out of memory\n"); + goto out; + } + + list_add_tail(&grp->grp_link, + &console_session.ses_grp_list); + lstcon_group_addref(grp); + } + + if (grp->grp_ref > 2) { + /* Group in using */ + jrep->join_status = EBUSY; + goto out; + } + + rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0); + if (rc == 0) { + jrep->join_status = EEXIST; + goto out; + } + + rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1); + if (rc != 0) { + CERROR("Out of memory\n"); + goto out; + } + + ndl->ndl_node->nd_state = LST_NODE_ACTIVE; + ndl->ndl_node->nd_timeout = console_session.ses_timeout; + + strcpy(jrep->join_session, console_session.ses_name); + jrep->join_timeout = console_session.ses_timeout; + jrep->join_status = 0; + +out: + if (grp != NULL) + lstcon_group_put(grp); + + mutex_up(&console_session.ses_mutex); + + return rc; +} + +srpc_service_t lstcon_acceptor_service = +{ + .sv_name = "join session", + .sv_handler = lstcon_acceptor_handle, + .sv_bulk_ready = NULL, + .sv_id = SRPC_SERVICE_JOIN, + .sv_concur = SFW_SERVICE_CONCURRENCY, +}; + +extern int lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_data *data); + +DECLARE_IOCTL_HANDLER(lstcon_ioctl_handler, lstcon_ioctl_entry); + +/* initialize console */ +int +lstcon_console_init(void) +{ + int i; + int n; + int rc; + + memset(&console_session, 0, sizeof(lstcon_session_t)); + + console_session.ses_id = LST_INVALID_SID; + console_session.ses_state = LST_SESSION_NONE; + console_session.ses_timeout = 0; + console_session.ses_force = 0; + console_session.ses_expired = 0; + console_session.ses_laststamp = cfs_time_current_sec(); + + init_mutex(&console_session.ses_mutex); + + CFS_INIT_LIST_HEAD(&console_session.ses_ndl_list); + CFS_INIT_LIST_HEAD(&console_session.ses_grp_list); + CFS_INIT_LIST_HEAD(&console_session.ses_bat_list); + CFS_INIT_LIST_HEAD(&console_session.ses_trans_list); + + LIBCFS_ALLOC(console_session.ses_ndl_hash, + sizeof(struct list_head) * LST_GLOBAL_HASHSIZE); + if (console_session.ses_ndl_hash == NULL) + return -ENOMEM; + + for (i = 0; i < LST_GLOBAL_HASHSIZE; i++) + CFS_INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]); + + rc = srpc_add_service(&lstcon_acceptor_service); + LASSERT (rc != -EBUSY); + if (rc != 0) { + LIBCFS_FREE(console_session.ses_ndl_hash, + sizeof(struct list_head) * LST_GLOBAL_HASHSIZE); + return rc; + } + + n = srpc_service_add_buffers(&lstcon_acceptor_service, SFW_POST_BUFFERS); + if (n != SFW_POST_BUFFERS) { + rc = -ENOMEM; + goto out; + } + + rc = libcfs_register_ioctl(&lstcon_ioctl_handler); + + if (rc == 0) { + lstcon_rpc_module_init(); + return 0; + } + +out: + srpc_shutdown_service(&lstcon_acceptor_service); + srpc_remove_service(&lstcon_acceptor_service); + + LIBCFS_FREE(console_session.ses_ndl_hash, + sizeof(struct list_head) * LST_GLOBAL_HASHSIZE); + + srpc_wait_service_shutdown(&lstcon_acceptor_service); + + return rc; +} + +int +lstcon_console_fini(void) +{ + int i; + + mutex_down(&console_session.ses_mutex); + + libcfs_deregister_ioctl(&lstcon_ioctl_handler); + + srpc_shutdown_service(&lstcon_acceptor_service); + srpc_remove_service(&lstcon_acceptor_service); + + if (console_session.ses_state != LST_SESSION_NONE) + lstcon_session_end(); + + lstcon_rpc_module_fini(); + + mutex_up(&console_session.ses_mutex); + + LASSERT (list_empty(&console_session.ses_ndl_list)); + LASSERT (list_empty(&console_session.ses_grp_list)); + LASSERT (list_empty(&console_session.ses_bat_list)); + LASSERT (list_empty(&console_session.ses_trans_list)); + + for (i = 0; i < LST_NODE_HASHSIZE; i++) { + LASSERT (list_empty(&console_session.ses_ndl_hash[i])); + } + + LIBCFS_FREE(console_session.ses_ndl_hash, + sizeof(struct list_head) * LST_GLOBAL_HASHSIZE); + + srpc_wait_service_shutdown(&lstcon_acceptor_service); + + return 0; +} + +#endif diff --git a/lnet/selftest/console.h b/lnet/selftest/console.h new file mode 100644 index 0000000..cc72930 --- /dev/null +++ b/lnet/selftest/console.h @@ -0,0 +1,190 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Author: Liang Zhen + * + * This file is part of Lustre, http://www.lustre.org + * + * kernel structure for LST console + */ + +#ifndef __LST_CONSOLE_H__ +#define __LST_CONSOLE_H__ + +#ifdef __KERNEL__ + +#include +#include +#include +#include +#include "selftest.h" +#include "conrpc.h" + +typedef struct lstcon_node { + lnet_process_id_t nd_id; /* id of the node */ + int nd_ref; /* reference count */ + int nd_state; /* state of the node */ + int nd_timeout; /* session timeout */ + cfs_time_t nd_stamp; /* timestamp of last replied RPC */ + struct lstcon_rpc nd_ping; /* ping rpc */ +} lstcon_node_t; /*** node descriptor */ + +typedef struct { + struct list_head ndl_link; /* chain on list */ + struct list_head ndl_hlink; /* chain on hash */ + lstcon_node_t *ndl_node; /* pointer to node */ +} lstcon_ndlink_t; /*** node link descriptor */ + +typedef struct { + struct list_head grp_link; /* chain on global group list */ + int grp_ref; /* reference count */ + int grp_nnode; /* # of nodes */ + char grp_name[LST_NAME_SIZE]; /* group name */ + + struct list_head grp_trans_list; /* transaction list */ + struct list_head grp_ndl_list; /* nodes list */ + struct list_head grp_ndl_hash[0];/* hash table for nodes */ +} lstcon_group_t; /*** (alias of nodes) group descriptor */ + +#define LST_BATCH_IDLE 0xB0 /* idle batch */ +#define LST_BATCH_RUNNING 0xB1 /* running batch */ + +typedef struct lstcon_tsb_hdr { + lst_bid_t tsb_id; /* batch ID */ + int tsb_index; /* test index */ +} lstcon_tsb_hdr_t; + +typedef struct { + lstcon_tsb_hdr_t bat_hdr; /* test_batch header */ + struct list_head bat_link; /* chain on session's batches list */ + int bat_ntest; /* # of test */ + int bat_state; /* state of the batch */ + int bat_arg; /* parameter for run|stop, timeout for run, force for stop */ + char bat_name[LST_NAME_SIZE]; /* name of batch */ + + struct list_head bat_test_list; /* list head of tests (lstcon_test_t) */ + struct list_head bat_trans_list; /* list head of transaction */ + struct list_head bat_cli_list; /* list head of client nodes (lstcon_node_t) */ + struct list_head *bat_cli_hash; /* hash table of client nodes */ + struct list_head bat_srv_list; /* list head of server nodes */ + struct list_head *bat_srv_hash; /* hash table of server nodes */ +} lstcon_batch_t; /*** (tests ) batch descritptor */ + +typedef struct lstcon_test { + lstcon_tsb_hdr_t tes_hdr; /* test batch header */ + struct list_head tes_link; /* chain on batch's tests list */ + lstcon_batch_t *tes_batch; /* pointer to batch */ + + int tes_type; /* type of the test, i.e: bulk, ping */ + int tes_stop_onerr; /* stop on error */ + int tes_oneside; /* one-sided test */ + int tes_concur; /* concurrency */ + int tes_loop; /* loop count */ + int tes_dist; /* nodes distribution of target group */ + int tes_span; /* nodes span of target group */ + int tes_cliidx; /* client index, used for RPC creating */ + + struct list_head tes_trans_list; /* transaction list */ + lstcon_group_t *tes_src_grp; /* group run the test */ + lstcon_group_t *tes_dst_grp; /* target group */ + + int tes_paramlen; /* test parameter length */ + char tes_param[0]; /* test parameter */ +} lstcon_test_t; /*** a single test descriptor */ + +#define LST_GLOBAL_HASHSIZE 503 /* global nodes hash table size */ +#define LST_NODE_HASHSIZE 239 /* node hash table (for batch or group) */ + +#define LST_SESSION_NONE 0x0 /* no session */ +#define LST_SESSION_ACTIVE 0x1 /* working session */ + +#define LST_CONSOLE_TIMEOUT 300 /* default console timeout */ + +typedef struct { + struct semaphore ses_mutex; /* lock for session, only one thread can enter session */ + lst_sid_t ses_id; /* global session id */ + int ses_key; /* local session key */ + int ses_state; /* state of session */ + int ses_timeout; /* timeout in seconds */ + time_t ses_laststamp; /* last operation stamp (seconds) */ + int ses_force:1; /* force creating */ + int ses_shutdown:1; /* session is shutting down */ + int ses_expired:1; /* console is timedout */ + __u64 ses_id_cookie; /* batch id cookie */ + char ses_name[LST_NAME_SIZE]; /* session name */ + lstcon_rpc_trans_t *ses_ping; /* session pinger */ + stt_timer_t ses_ping_timer; /* timer for pinger */ + lstcon_trans_stat_t ses_trans_stat; /* transaction stats */ + + struct list_head ses_trans_list; /* global list of transaction */ + struct list_head ses_grp_list; /* global list of groups */ + struct list_head ses_bat_list; /* global list of batches */ + struct list_head ses_ndl_list; /* global list of nodes */ + struct list_head *ses_ndl_hash; /* hash table of nodes */ + + spinlock_t ses_rpc_lock; /* serialize */ + int ses_rpc_pending; /* number of pending RPCs */ + struct list_head ses_rpc_list; /* list head of RPCs */ + struct list_head ses_rpc_freelist; /* idle console rpc */ +} lstcon_session_t; /*** session descriptor */ + +extern lstcon_session_t console_session; +static inline lstcon_trans_stat_t * +lstcon_trans_stat(void) +{ + return &console_session.ses_trans_stat; +} + +static inline struct list_head * +lstcon_id2hash (lnet_process_id_t id, struct list_head *hash) +{ + unsigned int idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE; + + return &hash[idx]; +} + +extern int lstcon_session_match(lst_sid_t sid); +extern int lstcon_session_new(char *name, int key, + int timeout, int flags, lst_sid_t *sid_up); +extern int lstcon_session_info(lst_sid_t *sid_up, int *key, + lstcon_ndlist_ent_t *entp, char *name_up, int len); +extern int lstcon_session_end(void); +extern int lstcon_session_debug(int timeout, struct list_head *result_up); +extern int lstcon_batch_debug(int timeout, char *name, + int client, struct list_head *result_up); +extern int lstcon_group_debug(int timeout, char *name, + struct list_head *result_up); +extern int lstcon_nodes_debug(int timeout, int nnd, lnet_process_id_t *nds_up, + struct list_head *result_up); +extern int lstcon_group_add(char *name); +extern int lstcon_group_del(char *name); +extern int lstcon_group_clean(char *name, int args); +extern int lstcon_group_refresh(char *name, struct list_head *result_up); +extern int lstcon_nodes_add(char *name, int nnd, lnet_process_id_t *nds_up, + struct list_head *result_up); +extern int lstcon_nodes_remove(char *name, int nnd, lnet_process_id_t *nds_up, + struct list_head *result_up); +extern int lstcon_group_info(char *name, lstcon_ndlist_ent_t *gent_up, + int *index_p, int *ndent_p, lstcon_node_ent_t *ndents_up); +extern int lstcon_group_list(int idx, int len, char *name_up); +extern int lstcon_batch_add(char *name); +extern int lstcon_batch_run(char *name, int timeout, struct list_head *result_up); +extern int lstcon_batch_stop(char *name, int force, struct list_head *result_up); +extern int lstcon_test_batch_query(char *name, int testidx, + int client, int timeout, + struct list_head *result_up); +extern int lstcon_batch_del(char *name); +extern int lstcon_batch_list(int idx, int namelen, char *name_up); +extern int lstcon_batch_info(char *name, lstcon_test_batch_ent_t *ent_up, + int server, int testidx, int *index_p, + int *ndent_p, lstcon_node_ent_t *dents_up); +extern int lstcon_group_stat(char *grp_name, int timeout, + struct list_head *result_up); +extern int lstcon_nodes_stat(int count, lnet_process_id_t *ids_up, + int timeout, struct list_head *result_up); +extern int lstcon_test_add(char *name, int type, int loop, int concur, + int dist, int span, char *src_name, char * dst_name, + void *param, int paramlen, struct list_head *result_up); +#endif + +#endif diff --git a/lnet/selftest/framework.c b/lnet/selftest/framework.c new file mode 100644 index 0000000..6a2487b --- /dev/null +++ b/lnet/selftest/framework.c @@ -0,0 +1,1661 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2001, 2002 Cluster File Systems, Inc. + * Authors: Isaac Huang + * Liang Zhen + */ + +#define DEBUG_SUBSYSTEM S_LNET + +#include +#include +#include + +#include "selftest.h" + + +static int session_timeout = 100; +CFS_MODULE_PARM(session_timeout, "i", int, 0444, + "test session timeout in seconds (100 by default, 0 == never)"); + +#define SFW_TEST_CONCURRENCY 128 +#define SFW_TEST_RPC_TIMEOUT 64 +#define SFW_CLIENT_RPC_TIMEOUT 64 /* in seconds */ +#define SFW_EXTRA_TEST_BUFFERS 8 /* tolerate buggy peers with extra buffers */ + +#define sfw_test_buffers(tsi) ((tsi)->tsi_loop + SFW_EXTRA_TEST_BUFFERS) + +#define sfw_unpack_id(id) \ +do { \ + __swab64s(&(id).nid); \ + __swab32s(&(id).pid); \ +} while (0) + +#define sfw_unpack_sid(sid) \ +do { \ + __swab64s(&(sid).ses_nid); \ + __swab64s(&(sid).ses_stamp); \ +} while (0) + +#define sfw_unpack_fw_counters(fc) \ +do { \ + __swab32s(&(fc).brw_errors); \ + __swab32s(&(fc).active_tests); \ + __swab32s(&(fc).active_batches); \ + __swab32s(&(fc).zombie_sessions); \ +} while (0) + +#define sfw_unpack_rpc_counters(rc) \ +do { \ + __swab32s(&(rc).errors); \ + __swab32s(&(rc).rpcs_sent); \ + __swab32s(&(rc).rpcs_rcvd); \ + __swab32s(&(rc).rpcs_dropped); \ + __swab32s(&(rc).rpcs_expired); \ + __swab64s(&(rc).bulk_get); \ + __swab64s(&(rc).bulk_put); \ +} while (0) + +#define sfw_unpack_lnet_counters(lc) \ +do { \ + __swab32s(&(lc).errors); \ + __swab32s(&(lc).msgs_max); \ + __swab32s(&(lc).msgs_alloc); \ + __swab32s(&(lc).send_count); \ + __swab32s(&(lc).recv_count); \ + __swab32s(&(lc).drop_count); \ + __swab32s(&(lc).route_count); \ + __swab64s(&(lc).send_length); \ + __swab64s(&(lc).recv_length); \ + __swab64s(&(lc).drop_length); \ + __swab64s(&(lc).route_length); \ +} while (0) + +#define sfw_test_active(t) (atomic_read(&(t)->tsi_nactive) != 0) +#define sfw_batch_active(b) (atomic_read(&(b)->bat_nactive) != 0) + +struct smoketest_framework { + struct list_head fw_zombie_rpcs; /* RPCs to be recycled */ + struct list_head fw_zombie_sessions; /* stopping sessions */ + struct list_head fw_tests; /* registered test cases */ + atomic_t fw_nzombies; /* # zombie sessions */ + spinlock_t fw_lock; /* serialise */ + sfw_session_t *fw_session; /* _the_ session */ + int fw_shuttingdown; /* shutdown in progress */ + srpc_server_rpc_t *fw_active_srpc; /* running RPC */ +} sfw_data; + +/* forward ref's */ +int sfw_stop_batch (sfw_batch_t *tsb, int force); +void sfw_destroy_session (sfw_session_t *sn); + +static inline sfw_test_case_t * +sfw_find_test_case(int id) +{ + sfw_test_case_t *tsc; + + LASSERT (id <= SRPC_SERVICE_MAX_ID); + LASSERT (id > SRPC_FRAMEWORK_SERVICE_MAX_ID); + + list_for_each_entry (tsc, &sfw_data.fw_tests, tsc_list) { + if (tsc->tsc_srv_service->sv_id == id) + return tsc; + } + + return NULL; +} + +static int +sfw_register_test (srpc_service_t *service, sfw_test_client_ops_t *cliops) +{ + sfw_test_case_t *tsc; + + if (sfw_find_test_case(service->sv_id) != NULL) { + CERROR ("Failed to register test %s (%d)\n", + service->sv_name, service->sv_id); + return -EEXIST; + } + + LIBCFS_ALLOC(tsc, sizeof(sfw_test_case_t)); + if (tsc == NULL) + return -ENOMEM; + + memset(tsc, 0, sizeof(sfw_test_case_t)); + tsc->tsc_cli_ops = cliops; + tsc->tsc_srv_service = service; + + list_add_tail(&tsc->tsc_list, &sfw_data.fw_tests); + return 0; +} + +void +sfw_add_session_timer (void) +{ + sfw_session_t *sn = sfw_data.fw_session; + stt_timer_t *timer = &sn->sn_timer; + + LASSERT (!sfw_data.fw_shuttingdown); + + if (sn == NULL || sn->sn_timeout == 0) + return; + + LASSERT (!sn->sn_timer_active); + + sn->sn_timer_active = 1; + timer->stt_expires = cfs_time_add(sn->sn_timeout, + cfs_time_current_sec()); + stt_add_timer(timer); + return; +} + +int +sfw_del_session_timer (void) +{ + sfw_session_t *sn = sfw_data.fw_session; + + if (sn == NULL || !sn->sn_timer_active) + return 0; + + LASSERT (sn->sn_timeout != 0); + + if (stt_del_timer(&sn->sn_timer)) { /* timer defused */ + sn->sn_timer_active = 0; + return 0; + } + +#ifndef __KERNEL__ + /* Racing is impossible in single-threaded userland selftest */ + LBUG(); +#endif + return EBUSY; /* racing with sfw_session_expired() */ +} + +/* called with sfw_data.fw_lock held */ +static void +sfw_deactivate_session (void) +{ + sfw_session_t *sn = sfw_data.fw_session; + int nactive = 0; + sfw_batch_t *tsb; + + if (sn == NULL) + return; + + LASSERT (!sn->sn_timer_active); + + sfw_data.fw_session = NULL; + atomic_inc(&sfw_data.fw_nzombies); + list_add(&sn->sn_list, &sfw_data.fw_zombie_sessions); + + list_for_each_entry (tsb, &sn->sn_batches, bat_list) { + if (sfw_batch_active(tsb)) { + nactive++; + sfw_stop_batch(tsb, 1); + } + } + + if (nactive != 0) + return; /* wait for active batches to stop */ + + list_del_init(&sn->sn_list); + spin_unlock(&sfw_data.fw_lock); + + sfw_destroy_session(sn); + + spin_lock(&sfw_data.fw_lock); + return; +} + +#ifndef __KERNEL__ + +int +sfw_session_removed (void) +{ + return (sfw_data.fw_session == NULL) ? 1 : 0; +} + +#endif + +void +sfw_session_expired (void *data) +{ + sfw_session_t *sn = data; + + spin_lock(&sfw_data.fw_lock); + + LASSERT (sn->sn_timer_active); + LASSERT (sn == sfw_data.fw_session); + + CWARN ("Session expired! sid: %s-"LPU64", name: %s\n", + libcfs_nid2str(sn->sn_id.ses_nid), + sn->sn_id.ses_stamp, &sn->sn_name[0]); + + sn->sn_timer_active = 0; + sfw_deactivate_session(); + + spin_unlock(&sfw_data.fw_lock); + return; +} + +static inline void +sfw_init_session (sfw_session_t *sn, lst_sid_t sid, const char *name) +{ + stt_timer_t *timer = &sn->sn_timer; + + memset(sn, 0, sizeof(sfw_session_t)); + CFS_INIT_LIST_HEAD(&sn->sn_list); + CFS_INIT_LIST_HEAD(&sn->sn_batches); + atomic_set(&sn->sn_brw_errors, 0); + strncpy(&sn->sn_name[0], name, LST_NAME_SIZE); + + sn->sn_timer_active = 0; + sn->sn_id = sid; + sn->sn_timeout = session_timeout; + + timer->stt_data = sn; + timer->stt_func = sfw_session_expired; + CFS_INIT_LIST_HEAD(&timer->stt_list); +} + +/* completion handler for incoming framework RPCs */ +void +sfw_server_rpc_done (srpc_server_rpc_t *rpc) +{ + srpc_service_t *sv = rpc->srpc_service; + int status = rpc->srpc_status; + + CDEBUG (D_NET, + "Incoming framework RPC done: " + "service %s, peer %s, status %s:%d\n", + sv->sv_name, libcfs_id2str(rpc->srpc_peer), + swi_state2str(rpc->srpc_wi.wi_state), + status); + + if (rpc->srpc_bulk != NULL) + sfw_free_pages(rpc); + return; +} + +void +sfw_client_rpc_fini (srpc_client_rpc_t *rpc) +{ + LASSERT (rpc->crpc_bulk.bk_niov == 0); + LASSERT (list_empty(&rpc->crpc_list)); + LASSERT (atomic_read(&rpc->crpc_refcount) == 0); +#ifndef __KERNEL__ + LASSERT (rpc->crpc_bulk.bk_pages == NULL); +#endif + + CDEBUG (D_NET, + "Outgoing framework RPC done: " + "service %d, peer %s, status %s:%d:%d\n", + rpc->crpc_service, libcfs_id2str(rpc->crpc_dest), + swi_state2str(rpc->crpc_wi.wi_state), + rpc->crpc_aborted, rpc->crpc_status); + + spin_lock(&sfw_data.fw_lock); + + /* my callers must finish all RPCs before shutting me down */ + LASSERT (!sfw_data.fw_shuttingdown); + list_add(&rpc->crpc_list, &sfw_data.fw_zombie_rpcs); + + spin_unlock(&sfw_data.fw_lock); + return; +} + +sfw_batch_t * +sfw_find_batch (lst_bid_t bid) +{ + sfw_session_t *sn = sfw_data.fw_session; + sfw_batch_t *bat; + + LASSERT (sn != NULL); + + list_for_each_entry (bat, &sn->sn_batches, bat_list) { + if (bat->bat_id.bat_id == bid.bat_id) + return bat; + } + + return NULL; +} + +sfw_batch_t * +sfw_bid2batch (lst_bid_t bid) +{ + sfw_session_t *sn = sfw_data.fw_session; + sfw_batch_t *bat; + + LASSERT (sn != NULL); + + bat = sfw_find_batch(bid); + if (bat != NULL) + return bat; + + LIBCFS_ALLOC(bat, sizeof(sfw_batch_t)); + if (bat == NULL) + return NULL; + + bat->bat_error = 0; + bat->bat_session = sn; + bat->bat_id = bid; + atomic_set(&bat->bat_nactive, 0); + CFS_INIT_LIST_HEAD(&bat->bat_tests); + + list_add_tail(&bat->bat_list, &sn->sn_batches); + return bat; +} + +int +sfw_get_stats (srpc_stat_reqst_t *request, srpc_stat_reply_t *reply) +{ + sfw_session_t *sn = sfw_data.fw_session; + sfw_counters_t *cnt = &reply->str_fw; + sfw_batch_t *bat; + + reply->str_sid = (sn == NULL) ? LST_INVALID_SID : sn->sn_id; + + if (request->str_sid.ses_nid == LNET_NID_ANY) { + reply->str_status = EINVAL; + return 0; + } + + if (sn == NULL || !sfw_sid_equal(request->str_sid, sn->sn_id)) { + reply->str_status = ESRCH; + return 0; + } + + LNET_LOCK(); + reply->str_lnet = the_lnet.ln_counters; + LNET_UNLOCK(); + + srpc_get_counters(&reply->str_rpc); + + cnt->brw_errors = atomic_read(&sn->sn_brw_errors); + cnt->zombie_sessions = atomic_read(&sfw_data.fw_nzombies); + + cnt->active_tests = cnt->active_batches = 0; + list_for_each_entry (bat, &sn->sn_batches, bat_list) { + int n = atomic_read(&bat->bat_nactive); + + if (n > 0) { + cnt->active_batches++; + cnt->active_tests += n; + } + } + + reply->str_status = 0; + return 0; +} + +int +sfw_make_session (srpc_mksn_reqst_t *request, srpc_mksn_reply_t *reply) +{ + sfw_session_t *sn = sfw_data.fw_session; + + if (request->mksn_sid.ses_nid == LNET_NID_ANY) { + reply->mksn_sid = (sn == NULL) ? LST_INVALID_SID : sn->sn_id; + reply->mksn_status = EINVAL; + return 0; + } + + if (sn != NULL && !request->mksn_force) { + reply->mksn_sid = sn->sn_id; + reply->mksn_status = EBUSY; + strncpy(&reply->mksn_name[0], &sn->sn_name[0], LST_NAME_SIZE); + return 0; + } + + LIBCFS_ALLOC(sn, sizeof(sfw_session_t)); + if (sn == NULL) { + CERROR ("Dropping RPC (mksn) under memory pressure.\n"); + return -ENOMEM; + } + + sfw_init_session(sn, request->mksn_sid, &request->mksn_name[0]); + + spin_lock(&sfw_data.fw_lock); + + sfw_deactivate_session(); + LASSERT (sfw_data.fw_session == NULL); + sfw_data.fw_session = sn; + + spin_unlock(&sfw_data.fw_lock); + + reply->mksn_status = 0; + reply->mksn_sid = sn->sn_id; + reply->mksn_timeout = sn->sn_timeout; + return 0; +} + +int +sfw_remove_session (srpc_rmsn_reqst_t *request, srpc_rmsn_reply_t *reply) +{ + sfw_session_t *sn = sfw_data.fw_session; + + reply->rmsn_sid = (sn == NULL) ? LST_INVALID_SID : sn->sn_id; + + if (request->rmsn_sid.ses_nid == LNET_NID_ANY) { + reply->rmsn_status = EINVAL; + return 0; + } + + if (sn == NULL || !sfw_sid_equal(request->rmsn_sid, sn->sn_id)) { + reply->rmsn_status = (sn == NULL) ? ESRCH : EBUSY; + return 0; + } + + spin_lock(&sfw_data.fw_lock); + sfw_deactivate_session(); + spin_unlock(&sfw_data.fw_lock); + + reply->rmsn_status = 0; + reply->rmsn_sid = LST_INVALID_SID; + LASSERT (sfw_data.fw_session == NULL); + return 0; +} + +int +sfw_debug_session (srpc_debug_reqst_t *request, srpc_debug_reply_t *reply) +{ + sfw_session_t *sn = sfw_data.fw_session; + + if (sn == NULL) { + reply->dbg_status = ESRCH; + reply->dbg_sid = LST_INVALID_SID; + return 0; + } + + reply->dbg_status = 0; + reply->dbg_sid = sn->sn_id; + reply->dbg_timeout = sn->sn_timeout; + strncpy(reply->dbg_name, &sn->sn_name[0], LST_NAME_SIZE); + + return 0; +} + +void +sfw_test_rpc_fini (srpc_client_rpc_t *rpc) +{ + sfw_test_unit_t *tsu = rpc->crpc_priv; + sfw_test_instance_t *tsi = tsu->tsu_instance; + + /* Called with hold of tsi->tsi_lock */ + LASSERT (list_empty(&rpc->crpc_list)); + list_add(&rpc->crpc_list, &tsi->tsi_free_rpcs); +} + +int +sfw_load_test (sfw_test_instance_t *tsi) +{ + sfw_test_case_t *tsc = sfw_find_test_case(tsi->tsi_service); + int nrequired = sfw_test_buffers(tsi); + int nposted; + + LASSERT (tsc != NULL); + + if (tsi->tsi_is_client) { + tsi->tsi_ops = tsc->tsc_cli_ops; + return 0; + } + + nposted = srpc_service_add_buffers(tsc->tsc_srv_service, nrequired); + if (nposted != nrequired) { + CWARN ("Failed to reserve enough buffers: " + "service %s, %d needed, %d reserved\n", + tsc->tsc_srv_service->sv_name, nrequired, nposted); + srpc_service_remove_buffers(tsc->tsc_srv_service, nposted); + return -ENOMEM; + } + + CDEBUG (D_NET, "Reserved %d buffers for test %s\n", + nposted, tsc->tsc_srv_service->sv_name); + return 0; +} + +void +sfw_unload_test (sfw_test_instance_t *tsi) +{ + sfw_test_case_t *tsc = sfw_find_test_case(tsi->tsi_service); + + LASSERT (tsc != NULL); + + if (!tsi->tsi_is_client) + srpc_service_remove_buffers(tsc->tsc_srv_service, + sfw_test_buffers(tsi)); + return; +} + +void +sfw_destroy_test_instance (sfw_test_instance_t *tsi) +{ + srpc_client_rpc_t *rpc; + sfw_test_unit_t *tsu; + + if (!tsi->tsi_is_client) + goto clean; + + tsi->tsi_ops->tso_fini(tsi); + + LASSERT (!tsi->tsi_stopping); + LASSERT (list_empty(&tsi->tsi_active_rpcs)); + LASSERT (!sfw_test_active(tsi)); + + while (!list_empty(&tsi->tsi_units)) { + tsu = list_entry(tsi->tsi_units.next, + sfw_test_unit_t, tsu_list); + list_del(&tsu->tsu_list); + LIBCFS_FREE(tsu, sizeof(*tsu)); + } + + while (!list_empty(&tsi->tsi_free_rpcs)) { + rpc = list_entry(tsi->tsi_free_rpcs.next, + srpc_client_rpc_t, crpc_list); + list_del(&rpc->crpc_list); + LIBCFS_FREE(rpc, srpc_client_rpc_size(rpc)); + } + +clean: + sfw_unload_test(tsi); + LIBCFS_FREE(tsi, sizeof(*tsi)); + return; +} + +void +sfw_destroy_batch (sfw_batch_t *tsb) +{ + sfw_test_instance_t *tsi; + + LASSERT (!sfw_batch_active(tsb)); + LASSERT (list_empty(&tsb->bat_list)); + + while (!list_empty(&tsb->bat_tests)) { + tsi = list_entry(tsb->bat_tests.next, + sfw_test_instance_t, tsi_list); + list_del_init(&tsi->tsi_list); + sfw_destroy_test_instance(tsi); + } + + LIBCFS_FREE(tsb, sizeof(sfw_batch_t)); + return; +} + +void +sfw_destroy_session (sfw_session_t *sn) +{ + sfw_batch_t *batch; + + LASSERT (list_empty(&sn->sn_list)); + LASSERT (sn != sfw_data.fw_session); + + while (!list_empty(&sn->sn_batches)) { + batch = list_entry(sn->sn_batches.next, + sfw_batch_t, bat_list); + list_del_init(&batch->bat_list); + sfw_destroy_batch(batch); + } + + LIBCFS_FREE(sn, sizeof(*sn)); + atomic_dec(&sfw_data.fw_nzombies); + return; +} + +void +sfw_unpack_test_req (srpc_msg_t *msg) +{ + srpc_test_reqst_t *req = &msg->msg_body.tes_reqst; + + LASSERT (msg->msg_type == SRPC_MSG_TEST_REQST); + LASSERT (req->tsr_is_client); + + if (msg->msg_magic == SRPC_MSG_MAGIC) + return; /* no flipping needed */ + + LASSERT (msg->msg_magic == __swab32(SRPC_MSG_MAGIC)); + + if (req->tsr_service == SRPC_SERVICE_BRW) { + test_bulk_req_t *bulk = &req->tsr_u.bulk; + + __swab32s(&bulk->blk_opc); + __swab32s(&bulk->blk_npg); + __swab32s(&bulk->blk_flags); + return; + } + + if (req->tsr_service == SRPC_SERVICE_PING) { + test_ping_req_t *ping = &req->tsr_u.ping; + + __swab32s(&ping->png_size); + __swab32s(&ping->png_flags); + return; + } + + LBUG (); + return; +} + +int +sfw_add_test_instance (sfw_batch_t *tsb, srpc_server_rpc_t *rpc) +{ + srpc_msg_t *msg = &rpc->srpc_reqstbuf->buf_msg; + srpc_test_reqst_t *req = &msg->msg_body.tes_reqst; + srpc_bulk_t *bk = rpc->srpc_bulk; + int ndest = req->tsr_ndest; + sfw_test_unit_t *tsu; + sfw_test_instance_t *tsi; + int i; + int rc; + + LIBCFS_ALLOC(tsi, sizeof(*tsi)); + if (tsi == NULL) { + CERROR ("Can't allocate test instance for batch: "LPU64"\n", + tsb->bat_id.bat_id); + return -ENOMEM; + } + + memset(tsi, 0, sizeof(*tsi)); + spin_lock_init(&tsi->tsi_lock); + atomic_set(&tsi->tsi_nactive, 0); + CFS_INIT_LIST_HEAD(&tsi->tsi_units); + CFS_INIT_LIST_HEAD(&tsi->tsi_free_rpcs); + CFS_INIT_LIST_HEAD(&tsi->tsi_active_rpcs); + + tsi->tsi_stopping = 0; + tsi->tsi_batch = tsb; + tsi->tsi_loop = req->tsr_loop; + tsi->tsi_concur = req->tsr_concur; + tsi->tsi_service = req->tsr_service; + tsi->tsi_is_client = !!(req->tsr_is_client); + tsi->tsi_stop_onerr = !!(req->tsr_stop_onerr); + + rc = sfw_load_test(tsi); + if (rc != 0) { + LIBCFS_FREE(tsi, sizeof(*tsi)); + return rc; + } + + LASSERT (!sfw_batch_active(tsb)); + + if (!tsi->tsi_is_client) { + /* it's test server, just add it to tsb */ + list_add_tail(&tsi->tsi_list, &tsb->bat_tests); + return 0; + } + + LASSERT (bk != NULL); +#ifndef __KERNEL__ + LASSERT (bk->bk_pages != NULL); +#endif + LASSERT (bk->bk_niov * SFW_ID_PER_PAGE >= ndest); + LASSERT (bk->bk_len >= sizeof(lnet_process_id_t) * ndest); + + sfw_unpack_test_req(msg); + memcpy(&tsi->tsi_u, &req->tsr_u, sizeof(tsi->tsi_u)); + + for (i = 0; i < ndest; i++) { + lnet_process_id_t *dests; + lnet_process_id_t id; + int j; + +#ifdef __KERNEL__ + dests = cfs_page_address(bk->bk_iovs[i / SFW_ID_PER_PAGE].kiov_page); + LASSERT (dests != NULL); /* my pages are within KVM always */ +#else + dests = cfs_page_address(bk->bk_pages[i / SFW_ID_PER_PAGE]); +#endif + id = dests[i % SFW_ID_PER_PAGE]; + if (msg->msg_magic != SRPC_MSG_MAGIC) + sfw_unpack_id(id); + + for (j = 0; j < tsi->tsi_concur; j++) { + LIBCFS_ALLOC(tsu, sizeof(sfw_test_unit_t)); + if (tsu == NULL) { + rc = -ENOMEM; + CERROR ("Can't allocate tsu for %d\n", + tsi->tsi_service); + goto error; + } + + tsu->tsu_dest = id; + tsu->tsu_instance = tsi; + tsu->tsu_private = NULL; + list_add_tail(&tsu->tsu_list, &tsi->tsi_units); + } + } + + rc = tsi->tsi_ops->tso_init(tsi); + if (rc == 0) { + list_add_tail(&tsi->tsi_list, &tsb->bat_tests); + return 0; + } + +error: + LASSERT (rc != 0); + sfw_destroy_test_instance(tsi); + return rc; +} + +static void +sfw_test_unit_done (sfw_test_unit_t *tsu) +{ + sfw_test_instance_t *tsi = tsu->tsu_instance; + sfw_batch_t *tsb = tsi->tsi_batch; + sfw_session_t *sn = tsb->bat_session; + + LASSERT (sfw_test_active(tsi)); + + if (!atomic_dec_and_test(&tsi->tsi_nactive)) + return; + + /* the test instance is done */ + spin_lock(&tsi->tsi_lock); + + tsi->tsi_stopping = 0; + + spin_unlock(&tsi->tsi_lock); + + spin_lock(&sfw_data.fw_lock); + + if (!atomic_dec_and_test(&tsb->bat_nactive) || /* tsb still active */ + sn == sfw_data.fw_session) { /* sn also active */ + spin_unlock(&sfw_data.fw_lock); + return; + } + + LASSERT (!list_empty(&sn->sn_list)); /* I'm a zombie! */ + + list_for_each_entry (tsb, &sn->sn_batches, bat_list) { + if (sfw_batch_active(tsb)) { + spin_unlock(&sfw_data.fw_lock); + return; + } + } + + list_del_init(&sn->sn_list); + spin_unlock(&sfw_data.fw_lock); + + sfw_destroy_session(sn); + return; +} + +void +sfw_test_rpc_done (srpc_client_rpc_t *rpc) +{ + sfw_test_unit_t *tsu = rpc->crpc_priv; + sfw_test_instance_t *tsi = tsu->tsu_instance; + int done = 0; + + if (rpc->crpc_status != 0 && tsu->tsu_error == 0 && + (rpc->crpc_status != -EINTR || !tsi->tsi_stopping)) + tsu->tsu_error = rpc->crpc_status; + + tsi->tsi_ops->tso_done_rpc(tsu, rpc); + + spin_lock(&tsi->tsi_lock); + + LASSERT (sfw_test_active(tsi)); + LASSERT (!list_empty(&rpc->crpc_list)); + + list_del_init(&rpc->crpc_list); + + /* batch is stopping or loop is done or get error */ + if (tsi->tsi_stopping || + tsu->tsu_loop == 0 || + (tsu->tsu_error != 0 && tsi->tsi_stop_onerr)) + done = 1; + + /* dec ref for poster */ + srpc_client_rpc_decref(rpc); + + spin_unlock(&tsi->tsi_lock); + + if (!done) { + swi_schedule_workitem(&tsu->tsu_worker); + return; + } + + sfw_test_unit_done(tsu); + return; +} + +int +sfw_create_test_rpc (sfw_test_unit_t *tsu, lnet_process_id_t peer, + int nblk, int blklen, srpc_client_rpc_t **rpcpp) +{ + srpc_client_rpc_t *rpc = NULL; + sfw_test_instance_t *tsi = tsu->tsu_instance; + + spin_lock(&tsi->tsi_lock); + + LASSERT (sfw_test_active(tsi)); + + if (!list_empty(&tsi->tsi_free_rpcs)) { + /* pick request from buffer */ + rpc = list_entry(tsi->tsi_free_rpcs.next, + srpc_client_rpc_t, crpc_list); + LASSERT (nblk == rpc->crpc_bulk.bk_niov); + list_del_init(&rpc->crpc_list); + + srpc_init_client_rpc(rpc, peer, tsi->tsi_service, nblk, + blklen, sfw_test_rpc_done, + sfw_test_rpc_fini, tsu); + } + + spin_unlock(&tsi->tsi_lock); + + if (rpc == NULL) + rpc = srpc_create_client_rpc(peer, tsi->tsi_service, nblk, + blklen, sfw_test_rpc_done, + sfw_test_rpc_fini, tsu); + if (rpc == NULL) { + CERROR ("Can't create rpc for test %d\n", tsi->tsi_service); + return -ENOMEM; + } + + *rpcpp = rpc; + return 0; +} + +int +sfw_run_test (swi_workitem_t *wi) +{ + sfw_test_unit_t *tsu = wi->wi_data; + sfw_test_instance_t *tsi = tsu->tsu_instance; + srpc_client_rpc_t *rpc = NULL; + + LASSERT (wi == &tsu->tsu_worker); + + tsu->tsu_error = tsi->tsi_ops->tso_prep_rpc(tsu, tsu->tsu_dest, &rpc); + if (tsu->tsu_error != 0) { + LASSERT (rpc == NULL); + goto test_done; + } + + LASSERT (rpc != NULL); + + spin_lock(&tsi->tsi_lock); + + if (tsi->tsi_stopping) { + list_add(&rpc->crpc_list, &tsi->tsi_free_rpcs); + spin_unlock(&tsi->tsi_lock); + goto test_done; + } + + if (tsu->tsu_loop > 0) + tsu->tsu_loop--; + + list_add_tail(&rpc->crpc_list, &tsi->tsi_active_rpcs); + spin_unlock(&tsi->tsi_lock); + + rpc->crpc_timeout = SFW_TEST_RPC_TIMEOUT; + + spin_lock(&rpc->crpc_lock); + srpc_post_rpc(rpc); + spin_unlock(&rpc->crpc_lock); + return 0; + +test_done: + /* + * No one can schedule me now since: + * - previous RPC, if any, has done and + * - no new RPC is initiated. + * - my batch is still active; no one can run it again now. + * Cancel pending schedules and prevent future schedule attempts: + */ + swi_kill_workitem(wi); + sfw_test_unit_done(tsu); + return 1; +} + +int +sfw_run_batch (sfw_batch_t *tsb) +{ + swi_workitem_t *wi; + sfw_test_unit_t *tsu; + sfw_test_instance_t *tsi; + + if (sfw_batch_active(tsb)) { + CDEBUG (D_NET, "Can't start active batch: "LPU64" (%d)\n", + tsb->bat_id.bat_id, atomic_read(&tsb->bat_nactive)); + return -EPERM; + } + + list_for_each_entry (tsi, &tsb->bat_tests, tsi_list) { + if (!tsi->tsi_is_client) /* skip server instances */ + continue; + + LASSERT (!tsi->tsi_stopping); + LASSERT (!sfw_test_active(tsi)); + + atomic_inc(&tsb->bat_nactive); + + list_for_each_entry (tsu, &tsi->tsi_units, tsu_list) { + atomic_inc(&tsi->tsi_nactive); + + tsu->tsu_error = 0; + tsu->tsu_loop = tsi->tsi_loop; + + wi = &tsu->tsu_worker; + swi_init_workitem(wi, tsu, sfw_run_test); + swi_schedule_workitem(wi); + } + } + + return 0; +} + +int +sfw_stop_batch (sfw_batch_t *tsb, int force) +{ + sfw_test_instance_t *tsi; + srpc_client_rpc_t *rpc; + + if (!sfw_batch_active(tsb)) + return -EPERM; + + list_for_each_entry (tsi, &tsb->bat_tests, tsi_list) { + spin_lock(&tsi->tsi_lock); + + if (!tsi->tsi_is_client || + !sfw_test_active(tsi) || tsi->tsi_stopping) { + spin_unlock(&tsi->tsi_lock); + continue; + } + + tsi->tsi_stopping = 1; + + if (!force) { + spin_unlock(&tsi->tsi_lock); + continue; + } + + /* abort launched rpcs in the test */ + list_for_each_entry (rpc, &tsi->tsi_active_rpcs, crpc_list) { + spin_lock(&rpc->crpc_lock); + + srpc_abort_rpc(rpc, -EINTR); + + spin_unlock(&rpc->crpc_lock); + } + + spin_unlock(&tsi->tsi_lock); + } + + return 0; +} + +int +sfw_query_batch (sfw_batch_t *tsb, int testidx, srpc_batch_reply_t *reply) +{ + sfw_test_instance_t *tsi; + + if (testidx < 0) + return -EINVAL; + + if (testidx == 0) { + reply->bar_active = atomic_read(&tsb->bat_nactive); + return 0; + } + + list_for_each_entry (tsi, &tsb->bat_tests, tsi_list) { + if (testidx-- > 1) + continue; + + reply->bar_active = atomic_read(&tsi->tsi_nactive); + return 0; + } + + return -ENOENT; +} + +void +sfw_free_pages (srpc_server_rpc_t *rpc) +{ + srpc_free_bulk(rpc->srpc_bulk); + rpc->srpc_bulk = NULL; +} + +int +sfw_alloc_pages (srpc_server_rpc_t *rpc, int npages, int sink) +{ + LASSERT (rpc->srpc_bulk == NULL); + LASSERT (npages > 0 && npages <= LNET_MAX_IOV); + + rpc->srpc_bulk = srpc_alloc_bulk(npages, sink); + if (rpc->srpc_bulk == NULL) return -ENOMEM; + + return 0; +} + +int +sfw_add_test (srpc_server_rpc_t *rpc) +{ + sfw_session_t *sn = sfw_data.fw_session; + srpc_test_reply_t *reply = &rpc->srpc_replymsg.msg_body.tes_reply; + srpc_test_reqst_t *request; + int rc; + sfw_batch_t *bat; + + request = &rpc->srpc_reqstbuf->buf_msg.msg_body.tes_reqst; + reply->tsr_sid = (sn == NULL) ? LST_INVALID_SID : sn->sn_id; + + if (request->tsr_loop == 0 || + request->tsr_concur == 0 || + request->tsr_sid.ses_nid == LNET_NID_ANY || + request->tsr_ndest > SFW_MAX_NDESTS || + (request->tsr_is_client && request->tsr_ndest == 0) || + request->tsr_concur > SFW_MAX_CONCUR || + request->tsr_service > SRPC_SERVICE_MAX_ID || + request->tsr_service <= SRPC_FRAMEWORK_SERVICE_MAX_ID) { + reply->tsr_status = EINVAL; + return 0; + } + + if (sn == NULL || !sfw_sid_equal(request->tsr_sid, sn->sn_id) || + sfw_find_test_case(request->tsr_service) == NULL) { + reply->tsr_status = ENOENT; + return 0; + } + + bat = sfw_bid2batch(request->tsr_bid); + if (bat == NULL) { + CERROR ("Dropping RPC (%s) from %s under memory pressure.\n", + rpc->srpc_service->sv_name, + libcfs_id2str(rpc->srpc_peer)); + return -ENOMEM; + } + + if (sfw_batch_active(bat)) { + reply->tsr_status = EBUSY; + return 0; + } + + if (request->tsr_is_client && rpc->srpc_bulk == NULL) { + /* rpc will be resumed later in sfw_bulk_ready */ + return sfw_alloc_pages(rpc, + sfw_id_pages(request->tsr_ndest), 1); + } + + rc = sfw_add_test_instance(bat, rpc); + CDEBUG (rc == 0 ? D_NET : D_WARNING, + "%s test: sv %d %s, loop %d, concur %d, ndest %d\n", + rc == 0 ? "Added" : "Failed to add", request->tsr_service, + request->tsr_is_client ? "client" : "server", + request->tsr_loop, request->tsr_concur, request->tsr_ndest); + + reply->tsr_status = (rc < 0) ? -rc : rc; + return 0; +} + +int +sfw_control_batch (srpc_batch_reqst_t *request, srpc_batch_reply_t *reply) +{ + sfw_session_t *sn = sfw_data.fw_session; + int rc = 0; + sfw_batch_t *bat; + + reply->bar_sid = (sn == NULL) ? LST_INVALID_SID : sn->sn_id; + + if (sn == NULL || !sfw_sid_equal(request->bar_sid, sn->sn_id)) { + reply->bar_status = ESRCH; + return 0; + } + + bat = sfw_find_batch(request->bar_bid); + if (bat == NULL) { + reply->bar_status = ENOENT; + return 0; + } + + switch (request->bar_opc) { + case SRPC_BATCH_OPC_RUN: + rc = sfw_run_batch(bat); + break; + + case SRPC_BATCH_OPC_STOP: + rc = sfw_stop_batch(bat, request->bar_arg); + break; + + case SRPC_BATCH_OPC_QUERY: + rc = sfw_query_batch(bat, request->bar_testidx, reply); + break; + + default: + return -EINVAL; /* drop it */ + } + + reply->bar_status = (rc < 0) ? -rc : rc; + return 0; +} + +int +sfw_handle_server_rpc (srpc_server_rpc_t *rpc) +{ + srpc_service_t *sv = rpc->srpc_service; + srpc_msg_t *reply = &rpc->srpc_replymsg; + srpc_msg_t *request = &rpc->srpc_reqstbuf->buf_msg; + int rc = 0; + + LASSERT (sfw_data.fw_active_srpc == NULL); + LASSERT (sv->sv_id <= SRPC_FRAMEWORK_SERVICE_MAX_ID); + + spin_lock(&sfw_data.fw_lock); + + if (sfw_data.fw_shuttingdown) { + spin_unlock(&sfw_data.fw_lock); + return -ESHUTDOWN; + } + + /* Remove timer to avoid racing with it or expiring active session */ + if (sfw_del_session_timer() != 0) { + CERROR ("Dropping RPC (%s) from %s: racing with expiry timer.", + sv->sv_name, libcfs_id2str(rpc->srpc_peer)); + spin_unlock(&sfw_data.fw_lock); + return -EAGAIN; + } + + sfw_data.fw_active_srpc = rpc; + spin_unlock(&sfw_data.fw_lock); + + sfw_unpack_message(request); + LASSERT (request->msg_type == srpc_service2request(sv->sv_id)); + + switch(sv->sv_id) { + default: + LBUG (); + case SRPC_SERVICE_TEST: + rc = sfw_add_test(rpc); + break; + + case SRPC_SERVICE_BATCH: + rc = sfw_control_batch(&request->msg_body.bat_reqst, + &reply->msg_body.bat_reply); + break; + + case SRPC_SERVICE_QUERY_STAT: + rc = sfw_get_stats(&request->msg_body.stat_reqst, + &reply->msg_body.stat_reply); + break; + + case SRPC_SERVICE_DEBUG: + rc = sfw_debug_session(&request->msg_body.dbg_reqst, + &reply->msg_body.dbg_reply); + break; + + case SRPC_SERVICE_MAKE_SESSION: + rc = sfw_make_session(&request->msg_body.mksn_reqst, + &reply->msg_body.mksn_reply); + break; + + case SRPC_SERVICE_REMOVE_SESSION: + rc = sfw_remove_session(&request->msg_body.rmsn_reqst, + &reply->msg_body.rmsn_reply); + break; + } + + rpc->srpc_done = sfw_server_rpc_done; + spin_lock(&sfw_data.fw_lock); + +#ifdef __KERNEL__ + if (!sfw_data.fw_shuttingdown) + sfw_add_session_timer(); +#else + LASSERT (!sfw_data.fw_shuttingdown); + sfw_add_session_timer(); +#endif + + sfw_data.fw_active_srpc = NULL; + spin_unlock(&sfw_data.fw_lock); + return rc; +} + +int +sfw_bulk_ready (srpc_server_rpc_t *rpc, int status) +{ + srpc_service_t *sv = rpc->srpc_service; + int rc; + + LASSERT (rpc->srpc_bulk != NULL); + LASSERT (sv->sv_id == SRPC_SERVICE_TEST); + LASSERT (sfw_data.fw_active_srpc == NULL); + LASSERT (rpc->srpc_reqstbuf->buf_msg.msg_body.tes_reqst.tsr_is_client); + + spin_lock(&sfw_data.fw_lock); + + if (status != 0) { + CERROR ("Bulk transfer failed for RPC: " + "service %s, peer %s, status %d\n", + sv->sv_name, libcfs_id2str(rpc->srpc_peer), status); + spin_unlock(&sfw_data.fw_lock); + return -EIO; + } + + if (sfw_data.fw_shuttingdown) { + spin_unlock(&sfw_data.fw_lock); + return -ESHUTDOWN; + } + + if (sfw_del_session_timer() != 0) { + CERROR ("Dropping RPC (%s) from %s: racing with expiry timer", + sv->sv_name, libcfs_id2str(rpc->srpc_peer)); + spin_unlock(&sfw_data.fw_lock); + return -EAGAIN; + } + + sfw_data.fw_active_srpc = rpc; + spin_unlock(&sfw_data.fw_lock); + + rc = sfw_add_test(rpc); + + spin_lock(&sfw_data.fw_lock); + +#ifdef __KERNEL__ + if (!sfw_data.fw_shuttingdown) + sfw_add_session_timer(); +#else + LASSERT (!sfw_data.fw_shuttingdown); + sfw_add_session_timer(); +#endif + + sfw_data.fw_active_srpc = NULL; + spin_unlock(&sfw_data.fw_lock); + return rc; +} + +srpc_client_rpc_t * +sfw_create_rpc (lnet_process_id_t peer, int service, + int nbulkiov, int bulklen, + void (*done) (srpc_client_rpc_t *), void *priv) +{ + srpc_client_rpc_t *rpc; + + spin_lock(&sfw_data.fw_lock); + + LASSERT (!sfw_data.fw_shuttingdown); + LASSERT (service <= SRPC_FRAMEWORK_SERVICE_MAX_ID); + + if (nbulkiov == 0 && !list_empty(&sfw_data.fw_zombie_rpcs)) { + rpc = list_entry(sfw_data.fw_zombie_rpcs.next, + srpc_client_rpc_t, crpc_list); + list_del(&rpc->crpc_list); + spin_unlock(&sfw_data.fw_lock); + + srpc_init_client_rpc(rpc, peer, service, 0, 0, + done, sfw_client_rpc_fini, priv); + return rpc; + } + + spin_unlock(&sfw_data.fw_lock); + + rpc = srpc_create_client_rpc(peer, service, nbulkiov, bulklen, done, + nbulkiov != 0 ? NULL : sfw_client_rpc_fini, + priv); + return rpc; +} + +void +sfw_unpack_message (srpc_msg_t *msg) +{ + if (msg->msg_magic == SRPC_MSG_MAGIC) + return; /* no flipping needed */ + + LASSERT (msg->msg_magic == __swab32(SRPC_MSG_MAGIC)); + + __swab32s(&msg->msg_type); + + if (msg->msg_type == SRPC_MSG_STAT_REQST) { + srpc_stat_reqst_t *req = &msg->msg_body.stat_reqst; + + __swab32s(&req->str_type); + __swab64s(&req->str_rpyid); + sfw_unpack_sid(req->str_sid); + return; + } + + if (msg->msg_type == SRPC_MSG_STAT_REPLY) { + srpc_stat_reply_t *rep = &msg->msg_body.stat_reply; + + __swab32s(&rep->str_status); + sfw_unpack_sid(rep->str_sid); + sfw_unpack_fw_counters(rep->str_fw); + sfw_unpack_rpc_counters(rep->str_rpc); + sfw_unpack_lnet_counters(rep->str_lnet); + return; + } + + if (msg->msg_type == SRPC_MSG_MKSN_REQST) { + srpc_mksn_reqst_t *req = &msg->msg_body.mksn_reqst; + + __swab64s(&req->mksn_rpyid); + __swab32s(&req->mksn_force); + sfw_unpack_sid(req->mksn_sid); + return; + } + + if (msg->msg_type == SRPC_MSG_MKSN_REPLY) { + srpc_mksn_reply_t *rep = &msg->msg_body.mksn_reply; + + __swab32s(&rep->mksn_status); + __swab32s(&rep->mksn_timeout); + sfw_unpack_sid(rep->mksn_sid); + return; + } + + if (msg->msg_type == SRPC_MSG_RMSN_REQST) { + srpc_rmsn_reqst_t *req = &msg->msg_body.rmsn_reqst; + + __swab64s(&req->rmsn_rpyid); + sfw_unpack_sid(req->rmsn_sid); + return; + } + + if (msg->msg_type == SRPC_MSG_RMSN_REPLY) { + srpc_rmsn_reply_t *rep = &msg->msg_body.rmsn_reply; + + __swab32s(&rep->rmsn_status); + sfw_unpack_sid(rep->rmsn_sid); + return; + } + + if (msg->msg_type == SRPC_MSG_DEBUG_REQST) { + srpc_debug_reqst_t *req = &msg->msg_body.dbg_reqst; + + __swab64s(&req->dbg_rpyid); + __swab32s(&req->dbg_flags); + sfw_unpack_sid(req->dbg_sid); + return; + } + + if (msg->msg_type == SRPC_MSG_DEBUG_REPLY) { + srpc_debug_reply_t *rep = &msg->msg_body.dbg_reply; + + __swab32s(&rep->dbg_nbatch); + __swab32s(&rep->dbg_timeout); + sfw_unpack_sid(rep->dbg_sid); + return; + } + + if (msg->msg_type == SRPC_MSG_BATCH_REQST) { + srpc_batch_reqst_t *req = &msg->msg_body.bat_reqst; + + __swab32s(&req->bar_opc); + __swab64s(&req->bar_rpyid); + __swab32s(&req->bar_testidx); + __swab32s(&req->bar_arg); + sfw_unpack_sid(req->bar_sid); + __swab64s(&req->bar_bid.bat_id); + return; + } + + if (msg->msg_type == SRPC_MSG_BATCH_REPLY) { + srpc_batch_reply_t *rep = &msg->msg_body.bat_reply; + + __swab32s(&rep->bar_status); + sfw_unpack_sid(rep->bar_sid); + return; + } + + if (msg->msg_type == SRPC_MSG_TEST_REQST) { + srpc_test_reqst_t *req = &msg->msg_body.tes_reqst; + + __swab64s(&req->tsr_rpyid); + __swab64s(&req->tsr_bulkid); + __swab32s(&req->tsr_loop); + __swab32s(&req->tsr_ndest); + __swab32s(&req->tsr_concur); + __swab32s(&req->tsr_service); + sfw_unpack_sid(req->tsr_sid); + __swab64s(&req->tsr_bid.bat_id); + return; + } + + if (msg->msg_type == SRPC_MSG_TEST_REPLY) { + srpc_test_reply_t *rep = &msg->msg_body.tes_reply; + + __swab32s(&rep->tsr_status); + sfw_unpack_sid(rep->tsr_sid); + return; + } + + if (msg->msg_type == SRPC_MSG_JOIN_REQST) { + srpc_join_reqst_t *req = &msg->msg_body.join_reqst; + + __swab64s(&req->join_rpyid); + sfw_unpack_sid(req->join_sid); + return; + } + + if (msg->msg_type == SRPC_MSG_JOIN_REPLY) { + srpc_join_reply_t *rep = &msg->msg_body.join_reply; + + __swab32s(&rep->join_status); + __swab32s(&rep->join_timeout); + sfw_unpack_sid(rep->join_sid); + return; + } + + LBUG (); + return; +} + +void +sfw_abort_rpc (srpc_client_rpc_t *rpc) +{ + LASSERT (atomic_read(&rpc->crpc_refcount) > 0); + LASSERT (rpc->crpc_service <= SRPC_FRAMEWORK_SERVICE_MAX_ID); + + spin_lock(&rpc->crpc_lock); + srpc_abort_rpc(rpc, -EINTR); + spin_unlock(&rpc->crpc_lock); + return; +} + +void +sfw_post_rpc (srpc_client_rpc_t *rpc) +{ + spin_lock(&rpc->crpc_lock); + + LASSERT (!rpc->crpc_closed); + LASSERT (!rpc->crpc_aborted); + LASSERT (list_empty(&rpc->crpc_list)); + LASSERT (!sfw_data.fw_shuttingdown); + + rpc->crpc_timeout = SFW_CLIENT_RPC_TIMEOUT; + srpc_post_rpc(rpc); + + spin_unlock(&rpc->crpc_lock); + return; +} + +static srpc_service_t sfw_services[] = +{ + { + .sv_name = "debug", + .sv_id = SRPC_SERVICE_DEBUG, + }, + { + .sv_name = "query stats", + .sv_id = SRPC_SERVICE_QUERY_STAT, + }, + { + .sv_name = "make sessin", + .sv_id = SRPC_SERVICE_MAKE_SESSION, + }, + { + .sv_name = "remove session", + .sv_id = SRPC_SERVICE_REMOVE_SESSION, + }, + { + .sv_name = "batch service", + .sv_id = SRPC_SERVICE_BATCH, + }, + { + .sv_name = "test service", + .sv_id = SRPC_SERVICE_TEST, + }, + { .sv_name = NULL, } +}; + +extern sfw_test_client_ops_t ping_test_client; +extern srpc_service_t ping_test_service; + +extern sfw_test_client_ops_t brw_test_client; +extern srpc_service_t brw_test_service; + +int +sfw_startup (void) +{ + int i; + int rc; + int error; + srpc_service_t *sv; + sfw_test_case_t *tsc; + + if (session_timeout < 0) { + CERROR ("Session timeout must be non-negative: %d\n", + session_timeout); + return -EINVAL; + } + + if (session_timeout == 0) + CWARN ("Zero session_timeout specified " + "- test sessions never timeout.\n"); + + memset(&sfw_data, 0, sizeof(struct smoketest_framework)); + + sfw_data.fw_session = NULL; + sfw_data.fw_active_srpc = NULL; + spin_lock_init(&sfw_data.fw_lock); + atomic_set(&sfw_data.fw_nzombies, 0); + CFS_INIT_LIST_HEAD(&sfw_data.fw_tests); + CFS_INIT_LIST_HEAD(&sfw_data.fw_zombie_rpcs); + CFS_INIT_LIST_HEAD(&sfw_data.fw_zombie_sessions); + + rc = sfw_register_test(&brw_test_service, &brw_test_client); + LASSERT (rc == 0); + rc = sfw_register_test(&ping_test_service, &ping_test_client); + LASSERT (rc == 0); + + error = 0; + list_for_each_entry (tsc, &sfw_data.fw_tests, tsc_list) { + sv = tsc->tsc_srv_service; + sv->sv_concur = SFW_TEST_CONCURRENCY; + + rc = srpc_add_service(sv); + LASSERT (rc != -EBUSY); + if (rc != 0) { + CWARN ("Failed to add %s service: %d\n", + sv->sv_name, rc); + error = rc; + } + } + + for (i = 0; ; i++) { + sv = &sfw_services[i]; + if (sv->sv_name == NULL) break; + + sv->sv_bulk_ready = NULL; + sv->sv_handler = sfw_handle_server_rpc; + sv->sv_concur = SFW_SERVICE_CONCURRENCY; + if (sv->sv_id == SRPC_SERVICE_TEST) + sv->sv_bulk_ready = sfw_bulk_ready; + + rc = srpc_add_service(sv); + LASSERT (rc != -EBUSY); + if (rc != 0) { + CWARN ("Failed to add %s service: %d\n", + sv->sv_name, rc); + error = rc; + } + + /* about to sfw_shutdown, no need to add buffer */ + if (error) continue; + + rc = srpc_service_add_buffers(sv, SFW_POST_BUFFERS); + if (rc != SFW_POST_BUFFERS) { + CWARN ("Failed to reserve enough buffers: " + "service %s, %d needed, %d reserved\n", + sv->sv_name, SFW_POST_BUFFERS, rc); + error = -ENOMEM; + } + } + + if (error != 0) + sfw_shutdown(); + return error; +} + +void +sfw_shutdown (void) +{ + srpc_service_t *sv; + sfw_test_case_t *tsc; + int i; + + spin_lock(&sfw_data.fw_lock); + + sfw_data.fw_shuttingdown = 1; +#ifdef __KERNEL__ + lst_wait_until(sfw_data.fw_active_srpc == NULL, sfw_data.fw_lock, + "waiting for active RPC to finish.\n"); +#else + LASSERT (sfw_data.fw_active_srpc == NULL); +#endif + + if (sfw_del_session_timer() != 0) + lst_wait_until(sfw_data.fw_session == NULL, sfw_data.fw_lock, + "waiting for session timer to explode.\n"); + + sfw_deactivate_session(); + lst_wait_until(atomic_read(&sfw_data.fw_nzombies) == 0, + sfw_data.fw_lock, + "waiting for %d zombie sessions to die.\n", + atomic_read(&sfw_data.fw_nzombies)); + + spin_unlock(&sfw_data.fw_lock); + + for (i = 0; ; i++) { + sv = &sfw_services[i]; + if (sv->sv_name == NULL) + break; + + srpc_shutdown_service(sv); + srpc_remove_service(sv); + } + + list_for_each_entry (tsc, &sfw_data.fw_tests, tsc_list) { + sv = tsc->tsc_srv_service; + srpc_shutdown_service(sv); + srpc_remove_service(sv); + } + + while (!list_empty(&sfw_data.fw_zombie_rpcs)) { + srpc_client_rpc_t *rpc; + + rpc = list_entry(sfw_data.fw_zombie_rpcs.next, + srpc_client_rpc_t, crpc_list); + list_del(&rpc->crpc_list); + + LIBCFS_FREE(rpc, srpc_client_rpc_size(rpc)); + } + + for (i = 0; ; i++) { + sv = &sfw_services[i]; + if (sv->sv_name == NULL) + break; + + srpc_wait_service_shutdown(sv); + } + + while (!list_empty(&sfw_data.fw_tests)) { + tsc = list_entry(sfw_data.fw_tests.next, + sfw_test_case_t, tsc_list); + + srpc_wait_service_shutdown(tsc->tsc_srv_service); + + list_del(&tsc->tsc_list); + LIBCFS_FREE(tsc, sizeof(*tsc)); + } + + return; +} diff --git a/lnet/selftest/module.c b/lnet/selftest/module.c new file mode 100644 index 0000000..5986acd --- /dev/null +++ b/lnet/selftest/module.c @@ -0,0 +1,106 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + */ +#define DEBUG_SUBSYSTEM S_LNET + +#include "selftest.h" + + +#define LST_INIT_NONE 0 +#define LST_INIT_RPC 1 +#define LST_INIT_FW 2 +#define LST_INIT_CONSOLE 3 + +extern int lstcon_console_init(void); +extern int lstcon_console_fini(void); + +static int lst_init_step = LST_INIT_NONE; + +void +lnet_selftest_fini (void) +{ + switch (lst_init_step) { +#ifdef __KERNEL__ + case LST_INIT_CONSOLE: + lstcon_console_fini(); +#endif + case LST_INIT_FW: + sfw_shutdown(); + case LST_INIT_RPC: + srpc_shutdown(); + case LST_INIT_NONE: + break; + default: + LBUG(); + } + return; +} + +int +lnet_selftest_init (void) +{ + int rc; + + rc = srpc_startup(); + if (rc != 0) { + CERROR("LST can't startup rpc\n"); + goto error; + } + lst_init_step = LST_INIT_RPC; + + rc = sfw_startup(); + if (rc != 0) { + CERROR("LST can't startup framework\n"); + goto error; + } + lst_init_step = LST_INIT_FW; + +#ifdef __KERNEL__ + rc = lstcon_console_init(); + if (rc != 0) { + CERROR("LST can't startup console\n"); + goto error; + } + lst_init_step = LST_INIT_CONSOLE; +#endif + + return 0; +error: + lnet_selftest_fini(); + return rc; +} + +#ifdef __KERNEL__ + +MODULE_DESCRIPTION("LNet Selftest"); +MODULE_LICENSE("GPL"); + +cfs_module(lnet, "0.9.0", lnet_selftest_init, lnet_selftest_fini); + +#else + +int +selftest_wait_events (void) +{ + int evts = 0; + + for (;;) { + /* Consume all pending events */ + while (srpc_check_event(0)) + evts++; + evts += stt_check_events(); + evts += swi_check_events(); + if (evts != 0) break; + + /* Nothing happened, block for events */ + evts += srpc_check_event(stt_poll_interval()); + /* We may have blocked, check for expired timers */ + evts += stt_check_events(); + if (evts == 0) /* timed out and still no event */ + break; + } + + return evts; +} + +#endif diff --git a/lnet/selftest/ping_test.c b/lnet/selftest/ping_test.c new file mode 100644 index 0000000..93fc9cf --- /dev/null +++ b/lnet/selftest/ping_test.c @@ -0,0 +1,162 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Author: Liang Zhen + * + * This file is part of Lustre, http://www.lustre.org + * + * Test client & Server + */ +#include +#include "selftest.h" + +#define LST_PING_TEST_MAGIC 0xbabeface + +typedef struct { + spinlock_t pnd_lock; /* serialize */ + int pnd_counter; /* sequence counter */ + int pnd_err_count; /* error count */ +} lst_ping_data_t; + +static lst_ping_data_t lst_ping_data; + +static int +ping_client_init(sfw_test_instance_t *tsi) +{ + spin_lock_init(&lst_ping_data.pnd_lock); + lst_ping_data.pnd_counter = 0; + lst_ping_data.pnd_err_count = 0; + + return 0; +} + +static void +ping_client_fini(sfw_test_instance_t *tsi) +{ + CWARN("Total ping %d, failed ping: %d\n", + lst_ping_data.pnd_counter, lst_ping_data.pnd_err_count); +} + +static int +ping_client_prep_rpc(sfw_test_unit_t *tsu, + lnet_process_id_t dest, srpc_client_rpc_t **rpc) +{ + srpc_ping_reqst_t *req; + struct timeval tv; + int rc; + + rc = sfw_create_test_rpc(tsu, dest, 0, 0, rpc); + if (rc != 0) + return rc; + + req = &(*rpc)->crpc_reqstmsg.msg_body.ping_reqst; + + req->pnr_magic = LST_PING_TEST_MAGIC; + + spin_lock(&lst_ping_data.pnd_lock); + req->pnr_seq = lst_ping_data.pnd_counter ++; + spin_unlock(&lst_ping_data.pnd_lock); + + cfs_fs_timeval(&tv); + req->pnr_time_sec = tv.tv_sec; + req->pnr_time_usec = tv.tv_usec; + + return rc; +} + +static void +ping_client_done_rpc(sfw_test_unit_t *tsu, srpc_client_rpc_t *rpc) +{ + srpc_ping_reqst_t *req; + srpc_ping_reply_t *rep; + struct timeval tv; + + req = &rpc->crpc_reqstmsg.msg_body.ping_reqst; + rep = &rpc->crpc_replymsg.msg_body.ping_reply; + + if (rpc->crpc_status == 0 && + rpc->crpc_replymsg.msg_magic != SRPC_MSG_MAGIC) { + __swab32s(&rep->pnr_seq); + __swab32s(&rep->pnr_magic); + __swab32s(&rep->pnr_status); + } + + if (rpc->crpc_status != 0) { + CERROR ("Unable to ping %s (%d): %d\n", + libcfs_id2str(rpc->crpc_dest), + req->pnr_seq, rpc->crpc_status); + } else if (rep->pnr_magic != LST_PING_TEST_MAGIC) { + tsu->tsu_error = -EBADMSG; + CERROR ("Bad magic %u from %s, %u expected.\n", + rep->pnr_magic, libcfs_id2str(rpc->crpc_dest), + LST_PING_TEST_MAGIC); + } else if (rep->pnr_seq != req->pnr_seq) { + tsu->tsu_error = -EBADMSG; + CERROR ("Bad seq %u from %s, %u expected.\n", + rep->pnr_seq, libcfs_id2str(rpc->crpc_dest), + req->pnr_seq); + } + + if (tsu->tsu_error != 0) { + spin_lock(&lst_ping_data.pnd_lock); + lst_ping_data.pnd_err_count++; + spin_unlock(&lst_ping_data.pnd_lock); + return; + } + + cfs_fs_timeval(&tv); + CDEBUG (D_NET, "%d reply in %u usec\n", rep->pnr_seq, + (unsigned)((tv.tv_sec - (unsigned)req->pnr_time_sec) * 1000000 + + (tv.tv_usec - req->pnr_time_usec))); + return; +} + +static int +ping_server_handle (srpc_server_rpc_t *rpc) +{ + srpc_service_t *sv = rpc->srpc_service; + srpc_msg_t *reqstmsg = &rpc->srpc_reqstbuf->buf_msg; + srpc_ping_reqst_t *req = &reqstmsg->msg_body.ping_reqst; + srpc_ping_reply_t *rep = &rpc->srpc_replymsg.msg_body.ping_reply; + + LASSERT (sv->sv_id == SRPC_SERVICE_PING); + + if (reqstmsg->msg_magic != SRPC_MSG_MAGIC) { + LASSERT (reqstmsg->msg_magic == __swab32(SRPC_MSG_MAGIC)); + + __swab32s(&reqstmsg->msg_type); + __swab32s(&req->pnr_seq); + __swab32s(&req->pnr_magic); + __swab64s(&req->pnr_time_sec); + __swab64s(&req->pnr_time_usec); + } + LASSERT (reqstmsg->msg_type == srpc_service2request(sv->sv_id)); + + if (req->pnr_magic != LST_PING_TEST_MAGIC) { + CERROR ("Unexpect magic %08x from %s\n", + req->pnr_magic, libcfs_id2str(rpc->srpc_peer)); + return -EINVAL; + } + + rep->pnr_seq = req->pnr_seq; + rep->pnr_magic = LST_PING_TEST_MAGIC; + + CDEBUG (D_NET, "Get ping %d from %s\n", + req->pnr_seq, libcfs_id2str(rpc->srpc_peer)); + return 0; +} + +sfw_test_client_ops_t ping_test_client = +{ + .tso_init = ping_client_init, + .tso_fini = ping_client_fini, + .tso_prep_rpc = ping_client_prep_rpc, + .tso_done_rpc = ping_client_done_rpc, +}; + +srpc_service_t ping_test_service = +{ + .sv_name = "ping test", + .sv_handler = ping_server_handle, + .sv_id = SRPC_SERVICE_PING, +}; diff --git a/lnet/selftest/rpc.c b/lnet/selftest/rpc.c new file mode 100644 index 0000000..bb6f587 --- /dev/null +++ b/lnet/selftest/rpc.c @@ -0,0 +1,1702 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2001, 2002 Cluster File Systems, Inc. + * Author: Isaac Huang + * + */ + +#define DEBUG_SUBSYSTEM S_LNET + +#include +#include +#include + +#include "selftest.h" + + +typedef enum { + SRPC_STATE_NONE, + SRPC_STATE_NI_INIT, + SRPC_STATE_EQ_INIT, + SRPC_STATE_WI_INIT, + SRPC_STATE_RUNNING, + SRPC_STATE_STOPPING, +} srpc_state_t; + +#define SRPC_PEER_HASH_SIZE 101 /* # peer lists */ +#define SRPC_PEER_CREDITS 16 /* >= most LND's default peer credit */ + +struct smoketest_rpc { + spinlock_t rpc_glock; /* global lock */ + srpc_service_t *rpc_services[SRPC_SERVICE_MAX_ID + 1]; + struct list_head *rpc_peers; /* hash table of known peers */ + lnet_handle_eq_t rpc_lnet_eq; /* _the_ LNet event queue */ + srpc_state_t rpc_state; + srpc_counters_t rpc_counters; + __u64 rpc_matchbits; /* matchbits counter */ +} srpc_data; + +/* forward ref's */ +int srpc_handle_rpc (swi_workitem_t *wi); + +void srpc_get_counters (srpc_counters_t *cnt) +{ + spin_lock(&srpc_data.rpc_glock); + *cnt = srpc_data.rpc_counters; + spin_unlock(&srpc_data.rpc_glock); +} + +void srpc_set_counters (const srpc_counters_t *cnt) +{ + spin_lock(&srpc_data.rpc_glock); + srpc_data.rpc_counters = *cnt; + spin_unlock(&srpc_data.rpc_glock); +} + +void +srpc_add_bulk_page (srpc_bulk_t *bk, cfs_page_t *pg, int i) +{ + LASSERT (i >= 0 && i < bk->bk_niov); + +#ifdef __KERNEL__ + bk->bk_iovs[i].kiov_offset = 0; + bk->bk_iovs[i].kiov_page = pg; + bk->bk_iovs[i].kiov_len = CFS_PAGE_SIZE; +#else + LASSERT (bk->bk_pages != NULL); + + bk->bk_pages[i] = pg; + bk->bk_iovs[i].iov_len = CFS_PAGE_SIZE; + bk->bk_iovs[i].iov_base = cfs_page_address(pg); +#endif + return; +} + +void +srpc_free_bulk (srpc_bulk_t *bk) +{ + int i; + cfs_page_t *pg; + + LASSERT (bk != NULL); +#ifndef __KERNEL__ + LASSERT (bk->bk_pages != NULL); +#endif + + for (i = 0; i < bk->bk_niov; i++) { +#ifdef __KERNEL__ + pg = bk->bk_iovs[i].kiov_page; +#else + pg = bk->bk_pages[i]; +#endif + if (pg == NULL) break; + + cfs_free_page(pg); + } + +#ifndef __KERNEL__ + LIBCFS_FREE(bk->bk_pages, sizeof(cfs_page_t *) * bk->bk_niov); +#endif + LIBCFS_FREE(bk, offsetof(srpc_bulk_t, bk_iovs[bk->bk_niov])); + return; +} + +srpc_bulk_t * +srpc_alloc_bulk (int npages, int sink) +{ + srpc_bulk_t *bk; + cfs_page_t **pages; + int i; + + LASSERT (npages > 0 && npages <= LNET_MAX_IOV); + + LIBCFS_ALLOC(bk, offsetof(srpc_bulk_t, bk_iovs[npages])); + if (bk == NULL) { + CERROR ("Can't allocate descriptor for %d pages\n", npages); + return NULL; + } + + memset(bk, 0, offsetof(srpc_bulk_t, bk_iovs[npages])); + bk->bk_sink = sink; + bk->bk_niov = npages; + bk->bk_len = npages * CFS_PAGE_SIZE; +#ifndef __KERNEL__ + LIBCFS_ALLOC(pages, sizeof(cfs_page_t *) * npages); + if (pages == NULL) { + LIBCFS_FREE(bk, offsetof(srpc_bulk_t, bk_iovs[npages])); + CERROR ("Can't allocate page array for %d pages\n", npages); + return NULL; + } + + memset(pages, 0, sizeof(cfs_page_t *) * npages); + bk->bk_pages = pages; +#else + UNUSED (pages); +#endif + + for (i = 0; i < npages; i++) { + cfs_page_t *pg = cfs_alloc_page(CFS_ALLOC_STD); + + if (pg == NULL) { + CERROR ("Can't allocate page %d of %d\n", i, npages); + srpc_free_bulk(bk); + return NULL; + } + + srpc_add_bulk_page(bk, pg, i); + } + + return bk; +} + + +static inline struct list_head * +srpc_nid2peerlist (lnet_nid_t nid) +{ + unsigned int hash = ((unsigned int)nid) % SRPC_PEER_HASH_SIZE; + + return &srpc_data.rpc_peers[hash]; +} + +static inline srpc_peer_t * +srpc_create_peer (lnet_nid_t nid) +{ + srpc_peer_t *peer; + + LASSERT (nid != LNET_NID_ANY); + + LIBCFS_ALLOC(peer, sizeof(srpc_peer_t)); + if (peer == NULL) { + CERROR ("Failed to allocate peer structure for %s\n", + libcfs_nid2str(nid)); + return NULL; + } + + memset(peer, 0, sizeof(srpc_peer_t)); + peer->stp_nid = nid; + peer->stp_credits = SRPC_PEER_CREDITS; + + spin_lock_init(&peer->stp_lock); + CFS_INIT_LIST_HEAD(&peer->stp_rpcq); + CFS_INIT_LIST_HEAD(&peer->stp_ctl_rpcq); + return peer; +} + +srpc_peer_t * +srpc_find_peer_locked (lnet_nid_t nid) +{ + struct list_head *peer_list = srpc_nid2peerlist(nid); + srpc_peer_t *peer; + + LASSERT (nid != LNET_NID_ANY); + + list_for_each_entry (peer, peer_list, stp_list) { + if (peer->stp_nid == nid) + return peer; + } + + return NULL; +} + +static srpc_peer_t * +srpc_nid2peer (lnet_nid_t nid) +{ + srpc_peer_t *peer; + srpc_peer_t *new_peer; + + spin_lock(&srpc_data.rpc_glock); + peer = srpc_find_peer_locked(nid); + spin_unlock(&srpc_data.rpc_glock); + + if (peer != NULL) + return peer; + + new_peer = srpc_create_peer(nid); + + spin_lock(&srpc_data.rpc_glock); + + peer = srpc_find_peer_locked(nid); + if (peer != NULL) { + spin_unlock(&srpc_data.rpc_glock); + if (new_peer != NULL) + LIBCFS_FREE(new_peer, sizeof(srpc_peer_t)); + + return peer; + } + + if (new_peer == NULL) { + spin_unlock(&srpc_data.rpc_glock); + return NULL; + } + + list_add_tail(&new_peer->stp_list, srpc_nid2peerlist(nid)); + spin_unlock(&srpc_data.rpc_glock); + return new_peer; +} + +static inline __u64 +srpc_next_id (void) +{ + __u64 id; + + spin_lock(&srpc_data.rpc_glock); + id = srpc_data.rpc_matchbits++; + spin_unlock(&srpc_data.rpc_glock); + return id; +} + +void +srpc_init_server_rpc (srpc_server_rpc_t *rpc, + srpc_service_t *sv, srpc_buffer_t *buffer) +{ + memset(rpc, 0, sizeof(*rpc)); + swi_init_workitem(&rpc->srpc_wi, rpc, srpc_handle_rpc); + + rpc->srpc_ev.ev_fired = 1; /* no event expected now */ + + rpc->srpc_service = sv; + rpc->srpc_reqstbuf = buffer; + rpc->srpc_peer = buffer->buf_peer; + rpc->srpc_self = buffer->buf_self; + rpc->srpc_replymdh = LNET_INVALID_HANDLE; +} + +int +srpc_add_service (srpc_service_t *sv) +{ + int id = sv->sv_id; + int i; + srpc_server_rpc_t *rpc; + + LASSERT (sv->sv_concur > 0); + LASSERT (0 <= id && id <= SRPC_SERVICE_MAX_ID); + + spin_lock(&srpc_data.rpc_glock); + + LASSERT (srpc_data.rpc_state == SRPC_STATE_RUNNING); + + if (srpc_data.rpc_services[id] != NULL) { + spin_unlock(&srpc_data.rpc_glock); + return -EBUSY; + } + + srpc_data.rpc_services[id] = sv; + spin_unlock(&srpc_data.rpc_glock); + + sv->sv_nprune = 0; + sv->sv_nposted_msg = 0; + sv->sv_shuttingdown = 0; + spin_lock_init(&sv->sv_lock); + CFS_INIT_LIST_HEAD(&sv->sv_free_rpcq); + CFS_INIT_LIST_HEAD(&sv->sv_active_rpcq); + CFS_INIT_LIST_HEAD(&sv->sv_posted_msgq); + CFS_INIT_LIST_HEAD(&sv->sv_blocked_msgq); + + sv->sv_ev.ev_data = sv; + sv->sv_ev.ev_type = SRPC_REQUEST_RCVD; + + for (i = 0; i < sv->sv_concur; i++) { + LIBCFS_ALLOC(rpc, sizeof(*rpc)); + if (rpc == NULL) goto enomem; + + list_add(&rpc->srpc_list, &sv->sv_free_rpcq); + } + + CDEBUG (D_NET, "Adding service: id %d, name %s, concurrency %d\n", + id, sv->sv_name, sv->sv_concur); + return 0; + +enomem: + while (!list_empty(&sv->sv_free_rpcq)) { + rpc = list_entry(sv->sv_free_rpcq.next, + srpc_server_rpc_t, srpc_list); + list_del(&rpc->srpc_list); + LIBCFS_FREE(rpc, sizeof(*rpc)); + } + + spin_lock(&srpc_data.rpc_glock); + srpc_data.rpc_services[id] = NULL; + spin_unlock(&srpc_data.rpc_glock); + return -ENOMEM; +} + +int +srpc_remove_service (srpc_service_t *sv) +{ + int id = sv->sv_id; + + spin_lock(&srpc_data.rpc_glock); + + if (srpc_data.rpc_services[id] != sv) { + spin_unlock(&srpc_data.rpc_glock); + return -ENOENT; + } + + srpc_data.rpc_services[id] = NULL; + spin_unlock(&srpc_data.rpc_glock); + return 0; +} + +int +srpc_post_passive_rdma(int portal, __u64 matchbits, void *buf, + int len, int options, lnet_process_id_t peer, + lnet_handle_md_t *mdh, srpc_event_t *ev) +{ + int rc; + lnet_md_t md; + lnet_handle_me_t meh; + + rc = LNetMEAttach(portal, peer, matchbits, 0, + LNET_UNLINK, LNET_INS_AFTER, &meh); + if (rc != 0) { + CERROR ("LNetMEAttach failed: %d\n", rc); + LASSERT (rc == -ENOMEM); + return -ENOMEM; + } + + md.threshold = 1; + md.user_ptr = ev; + md.start = buf; + md.length = len; + md.options = options; + md.eq_handle = srpc_data.rpc_lnet_eq; + + rc = LNetMDAttach(meh, md, LNET_UNLINK, mdh); + if (rc != 0) { + CERROR ("LNetMDAttach failed: %d\n", rc); + LASSERT (rc == -ENOMEM); + + rc = LNetMEUnlink(meh); + LASSERT (rc == 0); + return -ENOMEM; + } + + CDEBUG (D_NET, + "Posted passive RDMA: peer %s, portal %d, matchbits "LPX64"\n", + libcfs_id2str(peer), portal, matchbits); + return 0; +} + +int +srpc_post_active_rdma(int portal, __u64 matchbits, void *buf, int len, + int options, lnet_process_id_t peer, lnet_nid_t self, + lnet_handle_md_t *mdh, srpc_event_t *ev) +{ + int rc; + lnet_md_t md; + + md.user_ptr = ev; + md.start = buf; + md.length = len; + md.eq_handle = srpc_data.rpc_lnet_eq; + md.threshold = ((options & LNET_MD_OP_GET) != 0) ? 2 : 1; + md.options = options & ~(LNET_MD_OP_PUT | LNET_MD_OP_GET); + + rc = LNetMDBind(md, LNET_UNLINK, mdh); + if (rc != 0) { + CERROR ("LNetMDBind failed: %d\n", rc); + LASSERT (rc == -ENOMEM); + return -ENOMEM; + } + + /* this is kind of an abuse of the LNET_MD_OP_{PUT,GET} options. + * they're only meaningful for MDs attached to an ME (i.e. passive + * buffers... */ + if ((options & LNET_MD_OP_PUT) != 0) { + rc = LNetPut(self, *mdh, LNET_NOACK_REQ, peer, + portal, matchbits, 0, 0); + } else { + LASSERT ((options & LNET_MD_OP_GET) != 0); + + rc = LNetGet(self, *mdh, peer, portal, matchbits, 0); + } + + if (rc != 0) { + CERROR ("LNet%s(%s, %d, "LPD64") failed: %d\n", + ((options & LNET_MD_OP_PUT) != 0) ? "Put" : "Get", + libcfs_id2str(peer), portal, matchbits, rc); + + /* The forthcoming unlink event will complete this operation + * with failure, so fall through and return success here. + */ + rc = LNetMDUnlink(*mdh); + LASSERT (rc == 0); + } else { + CDEBUG (D_NET, + "Posted active RDMA: peer %s, portal %u, matchbits "LPX64"\n", + libcfs_id2str(peer), portal, matchbits); + } + return 0; +} + +int +srpc_post_active_rqtbuf(lnet_process_id_t peer, int service, void *buf, + int len, lnet_handle_md_t *mdh, srpc_event_t *ev) +{ + int rc; + int portal; + + if (service > SRPC_FRAMEWORK_SERVICE_MAX_ID) + portal = SRPC_REQUEST_PORTAL; + else + portal = SRPC_FRAMEWORK_REQUEST_PORTAL; + + rc = srpc_post_active_rdma(portal, service, buf, len, + LNET_MD_OP_PUT, peer, + LNET_NID_ANY, mdh, ev); + return rc; +} + +int +srpc_post_passive_rqtbuf(int service, void *buf, int len, + lnet_handle_md_t *mdh, srpc_event_t *ev) +{ + int rc; + int portal; + lnet_process_id_t any = {.nid = LNET_NID_ANY, + .pid = LNET_PID_ANY}; + + if (service > SRPC_FRAMEWORK_SERVICE_MAX_ID) + portal = SRPC_REQUEST_PORTAL; + else + portal = SRPC_FRAMEWORK_REQUEST_PORTAL; + + rc = srpc_post_passive_rdma(portal, service, buf, len, + LNET_MD_OP_PUT, any, mdh, ev); + return rc; +} + +int +srpc_service_post_buffer (srpc_service_t *sv, srpc_buffer_t *buf) +{ + srpc_msg_t *msg = &buf->buf_msg; + int rc; + + LASSERT (!sv->sv_shuttingdown); + + buf->buf_mdh = LNET_INVALID_HANDLE; + list_add(&buf->buf_list, &sv->sv_posted_msgq); + sv->sv_nposted_msg++; + spin_unlock(&sv->sv_lock); + + rc = srpc_post_passive_rqtbuf(sv->sv_id, msg, sizeof(*msg), + &buf->buf_mdh, &sv->sv_ev); + + /* At this point, a RPC (new or delayed) may have arrived in + * msg and its event handler has been called. So we must add + * buf to sv_posted_msgq _before_ dropping sv_lock */ + + spin_lock(&sv->sv_lock); + + if (rc == 0) { + if (sv->sv_shuttingdown) { + spin_unlock(&sv->sv_lock); + + /* srpc_shutdown_service might have tried to unlink me + * when my buf_mdh was still invalid */ + LNetMDUnlink(buf->buf_mdh); + + spin_lock(&sv->sv_lock); + } + return 0; + } + + sv->sv_nposted_msg--; + if (sv->sv_shuttingdown) return rc; + + list_del(&buf->buf_list); + + spin_unlock(&sv->sv_lock); + LIBCFS_FREE(buf, sizeof(*buf)); + spin_lock(&sv->sv_lock); + return rc; +} + +int +srpc_service_add_buffers (srpc_service_t *sv, int nbuffer) +{ + int rc; + int posted; + srpc_buffer_t *buf; + + LASSERTF (nbuffer > 0, + "nbuffer must be positive: %d\n", nbuffer); + + for (posted = 0; posted < nbuffer; posted++) { + LIBCFS_ALLOC(buf, sizeof(*buf)); + if (buf == NULL) break; + + spin_lock(&sv->sv_lock); + rc = srpc_service_post_buffer(sv, buf); + spin_unlock(&sv->sv_lock); + + if (rc != 0) break; + } + + return posted; +} + +void +srpc_service_remove_buffers (srpc_service_t *sv, int nbuffer) +{ + LASSERTF (nbuffer > 0, + "nbuffer must be positive: %d\n", nbuffer); + + spin_lock(&sv->sv_lock); + + LASSERT (sv->sv_nprune >= 0); + LASSERT (!sv->sv_shuttingdown); + + sv->sv_nprune += nbuffer; + + spin_unlock(&sv->sv_lock); + return; +} + +/* returns 1 if sv has finished, otherwise 0 */ +int +srpc_finish_service (srpc_service_t *sv) +{ + srpc_server_rpc_t *rpc; + srpc_buffer_t *buf; + + spin_lock(&sv->sv_lock); + + LASSERT (sv->sv_shuttingdown); /* srpc_shutdown_service called */ + + if (sv->sv_nposted_msg != 0 || !list_empty(&sv->sv_active_rpcq)) { + CDEBUG (D_NET, + "waiting for %d posted buffers to unlink and " + "in-flight RPCs to die.\n", + sv->sv_nposted_msg); + + if (!list_empty(&sv->sv_active_rpcq)) { + rpc = list_entry(sv->sv_active_rpcq.next, + srpc_server_rpc_t, srpc_list); + CDEBUG (D_NETERROR, + "Active RPC on shutdown: sv %s, peer %s, " + "wi %s scheduled %d running %d, " + "ev fired %d type %d status %d lnet %d\n", + sv->sv_name, libcfs_id2str(rpc->srpc_peer), + swi_state2str(rpc->srpc_wi.wi_state), + rpc->srpc_wi.wi_scheduled, + rpc->srpc_wi.wi_running, + rpc->srpc_ev.ev_fired, + rpc->srpc_ev.ev_type, + rpc->srpc_ev.ev_status, + rpc->srpc_ev.ev_lnet); + } + + spin_unlock(&sv->sv_lock); + return 0; + } + + spin_unlock(&sv->sv_lock); /* no lock needed from now on */ + + for (;;) { + struct list_head *q; + + if (!list_empty(&sv->sv_posted_msgq)) + q = &sv->sv_posted_msgq; + else if (!list_empty(&sv->sv_blocked_msgq)) + q = &sv->sv_blocked_msgq; + else + break; + + buf = list_entry(q->next, srpc_buffer_t, buf_list); + list_del(&buf->buf_list); + + LIBCFS_FREE(buf, sizeof(*buf)); + } + + while (!list_empty(&sv->sv_free_rpcq)) { + rpc = list_entry(sv->sv_free_rpcq.next, + srpc_server_rpc_t, srpc_list); + list_del(&rpc->srpc_list); + LIBCFS_FREE(rpc, sizeof(*rpc)); + } + + return 1; +} + +/* called with sv->sv_lock held */ +void +srpc_service_recycle_buffer (srpc_service_t *sv, srpc_buffer_t *buf) +{ + if (sv->sv_shuttingdown) + goto free; + + if (sv->sv_nprune == 0) { + if (srpc_service_post_buffer(sv, buf) != 0) + CWARN ("Failed to post %s buffer\n", sv->sv_name); + return; + } + + sv->sv_nprune--; +free: + spin_unlock(&sv->sv_lock); + LIBCFS_FREE(buf, sizeof(*buf)); + spin_lock(&sv->sv_lock); +} + +void +srpc_shutdown_service (srpc_service_t *sv) +{ + srpc_server_rpc_t *rpc; + srpc_buffer_t *buf; + + spin_lock(&sv->sv_lock); + + CDEBUG (D_NET, "Shutting down service: id %d, name %s\n", + sv->sv_id, sv->sv_name); + + sv->sv_shuttingdown = 1; /* i.e. no new active RPC */ + + /* schedule in-flight RPCs to notice the shutdown */ + list_for_each_entry (rpc, &sv->sv_active_rpcq, srpc_list) { + swi_schedule_workitem(&rpc->srpc_wi); + } + + spin_unlock(&sv->sv_lock); + + /* OK to traverse sv_posted_msgq without lock, since no one + * touches sv_posted_msgq now */ + list_for_each_entry (buf, &sv->sv_posted_msgq, buf_list) + LNetMDUnlink(buf->buf_mdh); + + return; +} + +int +srpc_send_request (srpc_client_rpc_t *rpc) +{ + srpc_event_t *ev = &rpc->crpc_reqstev; + int rc; + + ev->ev_fired = 0; + ev->ev_data = rpc; + ev->ev_type = SRPC_REQUEST_SENT; + + rc = srpc_post_active_rqtbuf(rpc->crpc_dest, rpc->crpc_service, + &rpc->crpc_reqstmsg, sizeof(srpc_msg_t), + &rpc->crpc_reqstmdh, ev); + if (rc != 0) { + LASSERT (rc == -ENOMEM); + ev->ev_fired = 1; /* no more event expected */ + } + return rc; +} + +int +srpc_prepare_reply (srpc_client_rpc_t *rpc) +{ + srpc_event_t *ev = &rpc->crpc_replyev; + __u64 *id = &rpc->crpc_reqstmsg.msg_body.reqst.rpyid; + int rc; + + ev->ev_fired = 0; + ev->ev_data = rpc; + ev->ev_type = SRPC_REPLY_RCVD; + + *id = srpc_next_id(); + + rc = srpc_post_passive_rdma(SRPC_RDMA_PORTAL, *id, + &rpc->crpc_replymsg, sizeof(srpc_msg_t), + LNET_MD_OP_PUT, rpc->crpc_dest, + &rpc->crpc_replymdh, ev); + if (rc != 0) { + LASSERT (rc == -ENOMEM); + ev->ev_fired = 1; /* no more event expected */ + } + return rc; +} + +int +srpc_prepare_bulk (srpc_client_rpc_t *rpc) +{ + srpc_bulk_t *bk = &rpc->crpc_bulk; + srpc_event_t *ev = &rpc->crpc_bulkev; + __u64 *id = &rpc->crpc_reqstmsg.msg_body.reqst.bulkid; + int rc; + int opt; + + LASSERT (bk->bk_niov <= LNET_MAX_IOV); + + if (bk->bk_niov == 0) return 0; /* nothing to do */ + + opt = bk->bk_sink ? LNET_MD_OP_PUT : LNET_MD_OP_GET; +#ifdef __KERNEL__ + opt |= LNET_MD_KIOV; +#else + opt |= LNET_MD_IOVEC; +#endif + + ev->ev_fired = 0; + ev->ev_data = rpc; + ev->ev_type = SRPC_BULK_REQ_RCVD; + + *id = srpc_next_id(); + + rc = srpc_post_passive_rdma(SRPC_RDMA_PORTAL, *id, + &bk->bk_iovs[0], bk->bk_niov, opt, + rpc->crpc_dest, &bk->bk_mdh, ev); + if (rc != 0) { + LASSERT (rc == -ENOMEM); + ev->ev_fired = 1; /* no more event expected */ + } + return rc; +} + +int +srpc_do_bulk (srpc_server_rpc_t *rpc) +{ + srpc_event_t *ev = &rpc->srpc_ev; + srpc_bulk_t *bk = rpc->srpc_bulk; + __u64 id = rpc->srpc_reqstbuf->buf_msg.msg_body.reqst.bulkid; + int rc; + int opt; + + LASSERT (bk != NULL); + + opt = bk->bk_sink ? LNET_MD_OP_GET : LNET_MD_OP_PUT; +#ifdef __KERNEL__ + opt |= LNET_MD_KIOV; +#else + opt |= LNET_MD_IOVEC; +#endif + + ev->ev_fired = 0; + ev->ev_data = rpc; + ev->ev_type = bk->bk_sink ? SRPC_BULK_GET_RPLD : SRPC_BULK_PUT_SENT; + + rc = srpc_post_active_rdma(SRPC_RDMA_PORTAL, id, + &bk->bk_iovs[0], bk->bk_niov, opt, + rpc->srpc_peer, rpc->srpc_self, + &bk->bk_mdh, ev); + if (rc != 0) + ev->ev_fired = 1; /* no more event expected */ + return rc; +} + +/* called with srpc_service_t::sv_lock held */ +inline void +srpc_schedule_server_rpc (srpc_server_rpc_t *rpc) +{ + srpc_service_t *sv = rpc->srpc_service; + + if (sv->sv_id > SRPC_FRAMEWORK_SERVICE_MAX_ID) + swi_schedule_workitem(&rpc->srpc_wi); + else /* framework RPCs are handled one by one */ + swi_schedule_serial_workitem(&rpc->srpc_wi); + + return; +} + +/* only called from srpc_handle_rpc */ +void +srpc_server_rpc_done (srpc_server_rpc_t *rpc, int status) +{ + srpc_service_t *sv = rpc->srpc_service; + srpc_buffer_t *buffer; + + LASSERT (status != 0 || rpc->srpc_wi.wi_state == SWI_STATE_DONE); + + rpc->srpc_status = status; + + CDEBUG (status == 0 ? D_NET : D_NETERROR, + "Server RPC done: service %s, peer %s, status %s:%d\n", + sv->sv_name, libcfs_id2str(rpc->srpc_peer), + swi_state2str(rpc->srpc_wi.wi_state), status); + + if (status != 0) { + spin_lock(&srpc_data.rpc_glock); + srpc_data.rpc_counters.rpcs_dropped++; + spin_unlock(&srpc_data.rpc_glock); + } + + if (rpc->srpc_done != NULL) + (*rpc->srpc_done) (rpc); + LASSERT (rpc->srpc_bulk == NULL); + + spin_lock(&sv->sv_lock); + + if (rpc->srpc_reqstbuf != NULL) { + /* NB might drop sv_lock in srpc_service_recycle_buffer, but + * sv won't go away for sv_active_rpcq must not be empty */ + srpc_service_recycle_buffer(sv, rpc->srpc_reqstbuf); + rpc->srpc_reqstbuf = NULL; + } + + list_del(&rpc->srpc_list); /* from sv->sv_active_rpcq */ + + /* + * No one can schedule me now since: + * - I'm not on sv_active_rpcq. + * - all LNet events have been fired. + * Cancel pending schedules and prevent future schedule attempts: + */ + LASSERT (rpc->srpc_ev.ev_fired); + swi_kill_workitem(&rpc->srpc_wi); + + if (!sv->sv_shuttingdown && !list_empty(&sv->sv_blocked_msgq)) { + buffer = list_entry(sv->sv_blocked_msgq.next, + srpc_buffer_t, buf_list); + list_del(&buffer->buf_list); + + srpc_init_server_rpc(rpc, sv, buffer); + list_add_tail(&rpc->srpc_list, &sv->sv_active_rpcq); + srpc_schedule_server_rpc(rpc); + } else { + list_add(&rpc->srpc_list, &sv->sv_free_rpcq); + } + + spin_unlock(&sv->sv_lock); + return; +} + +/* handles an incoming RPC */ +int +srpc_handle_rpc (swi_workitem_t *wi) +{ + srpc_server_rpc_t *rpc = wi->wi_data; + srpc_service_t *sv = rpc->srpc_service; + srpc_event_t *ev = &rpc->srpc_ev; + int rc = 0; + + LASSERT (wi == &rpc->srpc_wi); + + spin_lock(&sv->sv_lock); + + if (sv->sv_shuttingdown) { + spin_unlock(&sv->sv_lock); + + if (rpc->srpc_bulk != NULL) + LNetMDUnlink(rpc->srpc_bulk->bk_mdh); + LNetMDUnlink(rpc->srpc_replymdh); + + if (ev->ev_fired) { /* no more event, OK to finish */ + srpc_server_rpc_done(rpc, -ESHUTDOWN); + return 1; + } + return 0; + } + + spin_unlock(&sv->sv_lock); + + switch (wi->wi_state) { + default: + LBUG (); + case SWI_STATE_NEWBORN: { + srpc_msg_t *msg; + srpc_generic_reply_t *reply; + + msg = &rpc->srpc_reqstbuf->buf_msg; + reply = &rpc->srpc_replymsg.msg_body.reply; + + if (msg->msg_version != SRPC_MSG_VERSION && + msg->msg_version != __swab32(SRPC_MSG_VERSION)) { + CWARN ("Version mismatch: %u, %u expected, from %s\n", + msg->msg_version, SRPC_MSG_VERSION, + libcfs_id2str(rpc->srpc_peer)); + reply->status = EPROTO; + } else { + reply->status = 0; + rc = (*sv->sv_handler) (rpc); + LASSERT (reply->status == 0 || !rpc->srpc_bulk); + } + + if (rc != 0) { + srpc_server_rpc_done(rpc, rc); + return 1; + } + + wi->wi_state = SWI_STATE_BULK_STARTED; + + if (rpc->srpc_bulk != NULL) { + rc = srpc_do_bulk(rpc); + if (rc == 0) + return 0; /* wait for bulk */ + + LASSERT (ev->ev_fired); + ev->ev_status = rc; + } + } + case SWI_STATE_BULK_STARTED: + LASSERT (rpc->srpc_bulk == NULL || ev->ev_fired); + + if (rpc->srpc_bulk != NULL) { + rc = ev->ev_status; + + if (sv->sv_bulk_ready != NULL) + rc = (*sv->sv_bulk_ready) (rpc, rc); + + if (rc != 0) { + srpc_server_rpc_done(rpc, rc); + return 1; + } + } + + wi->wi_state = SWI_STATE_REPLY_SUBMITTED; + rc = srpc_send_reply(rpc); + if (rc == 0) + return 0; /* wait for reply */ + srpc_server_rpc_done(rpc, rc); + return 1; + + case SWI_STATE_REPLY_SUBMITTED: + LASSERT (ev->ev_fired); + + wi->wi_state = SWI_STATE_DONE; + srpc_server_rpc_done(rpc, ev->ev_status); + return 1; + } + + return 0; +} + +void +srpc_client_rpc_expired (void *data) +{ + srpc_client_rpc_t *rpc = data; + + CWARN ("Client RPC expired: service %d, peer %s, timeout %d.\n", + rpc->crpc_service, libcfs_id2str(rpc->crpc_dest), + rpc->crpc_timeout); + + spin_lock(&rpc->crpc_lock); + + rpc->crpc_timeout = 0; + srpc_abort_rpc(rpc, -ETIMEDOUT); + + spin_unlock(&rpc->crpc_lock); + + spin_lock(&srpc_data.rpc_glock); + srpc_data.rpc_counters.rpcs_expired++; + spin_unlock(&srpc_data.rpc_glock); + return; +} + +inline void +srpc_add_client_rpc_timer (srpc_client_rpc_t *rpc) +{ + stt_timer_t *timer = &rpc->crpc_timer; + + if (rpc->crpc_timeout == 0) return; + + CFS_INIT_LIST_HEAD(&timer->stt_list); + timer->stt_data = rpc; + timer->stt_func = srpc_client_rpc_expired; + timer->stt_expires = cfs_time_add(rpc->crpc_timeout, + cfs_time_current_sec()); + stt_add_timer(timer); + return; +} + +/* + * Called with rpc->crpc_lock held. + * + * Upon exit the RPC expiry timer is not queued and the handler is not + * running on any CPU. */ +void +srpc_del_client_rpc_timer (srpc_client_rpc_t *rpc) +{ + /* timer not planted or already exploded */ + if (rpc->crpc_timeout == 0) return; + + /* timer sucessfully defused */ + if (stt_del_timer(&rpc->crpc_timer)) return; + +#ifdef __KERNEL__ + /* timer detonated, wait for it to explode */ + while (rpc->crpc_timeout != 0) { + spin_unlock(&rpc->crpc_lock); + + cfs_schedule(); + + spin_lock(&rpc->crpc_lock); + } +#else + LBUG(); /* impossible in single-threaded runtime */ +#endif + return; +} + +void +srpc_check_sends (srpc_peer_t *peer, int credits) +{ + struct list_head *q; + srpc_client_rpc_t *rpc; + + LASSERT (credits >= 0); + LASSERT (srpc_data.rpc_state == SRPC_STATE_RUNNING); + + spin_lock(&peer->stp_lock); + peer->stp_credits += credits; + + while (peer->stp_credits) { + if (!list_empty(&peer->stp_ctl_rpcq)) + q = &peer->stp_ctl_rpcq; + else if (!list_empty(&peer->stp_rpcq)) + q = &peer->stp_rpcq; + else + break; + + peer->stp_credits--; + + rpc = list_entry(q->next, srpc_client_rpc_t, crpc_privl); + list_del_init(&rpc->crpc_privl); + srpc_client_rpc_decref(rpc); /* --ref for peer->*rpcq */ + + swi_schedule_workitem(&rpc->crpc_wi); + } + + spin_unlock(&peer->stp_lock); + return; +} + +void +srpc_client_rpc_done (srpc_client_rpc_t *rpc, int status) +{ + swi_workitem_t *wi = &rpc->crpc_wi; + srpc_peer_t *peer = rpc->crpc_peer; + + LASSERT (status != 0 || wi->wi_state == SWI_STATE_DONE); + + spin_lock(&rpc->crpc_lock); + + rpc->crpc_closed = 1; + if (rpc->crpc_status == 0) + rpc->crpc_status = status; + + srpc_del_client_rpc_timer(rpc); + + CDEBUG ((status == 0) ? D_NET : D_NETERROR, + "Client RPC done: service %d, peer %s, status %s:%d:%d\n", + rpc->crpc_service, libcfs_id2str(rpc->crpc_dest), + swi_state2str(wi->wi_state), rpc->crpc_aborted, status); + + /* + * No one can schedule me now since: + * - RPC timer has been defused. + * - all LNet events have been fired. + * - crpc_closed has been set, preventing srpc_abort_rpc from + * scheduling me. + * Cancel pending schedules and prevent future schedule attempts: + */ + LASSERT (!srpc_event_pending(rpc)); + swi_kill_workitem(wi); + + spin_unlock(&rpc->crpc_lock); + + (*rpc->crpc_done) (rpc); + + if (peer != NULL) + srpc_check_sends(peer, 1); + return; +} + +/* sends an outgoing RPC */ +int +srpc_send_rpc (swi_workitem_t *wi) +{ + int rc = 0; + srpc_client_rpc_t *rpc = wi->wi_data; + srpc_msg_t *reply = &rpc->crpc_replymsg; + int do_bulk = rpc->crpc_bulk.bk_niov > 0; + + LASSERT (rpc != NULL); + LASSERT (wi == &rpc->crpc_wi); + + spin_lock(&rpc->crpc_lock); + + if (rpc->crpc_aborted) { + spin_unlock(&rpc->crpc_lock); + goto abort; + } + + spin_unlock(&rpc->crpc_lock); + + switch (wi->wi_state) { + default: + LBUG (); + case SWI_STATE_NEWBORN: + LASSERT (!srpc_event_pending(rpc)); + + rc = srpc_prepare_reply(rpc); + if (rc != 0) { + srpc_client_rpc_done(rpc, rc); + return 1; + } + + rc = srpc_prepare_bulk(rpc); + if (rc != 0) break; + + wi->wi_state = SWI_STATE_REQUEST_SUBMITTED; + rc = srpc_send_request(rpc); + break; + + case SWI_STATE_REQUEST_SUBMITTED: + /* CAVEAT EMPTOR: rqtev, rpyev, and bulkev may come in any + * order; however, they're processed in a strict order: + * rqt, rpy, and bulk. */ + if (!rpc->crpc_reqstev.ev_fired) break; + + rc = rpc->crpc_reqstev.ev_status; + if (rc != 0) break; + + wi->wi_state = SWI_STATE_REQUEST_SENT; + /* perhaps more events, fall thru */ + case SWI_STATE_REQUEST_SENT: { + srpc_msg_type_t type = srpc_service2reply(rpc->crpc_service); + + if (!rpc->crpc_replyev.ev_fired) break; + + rc = rpc->crpc_replyev.ev_status; + if (rc != 0) break; + + if ((reply->msg_type != type && + reply->msg_type != __swab32(type)) || + (reply->msg_magic != SRPC_MSG_MAGIC && + reply->msg_magic != __swab32(SRPC_MSG_MAGIC))) { + CWARN ("Bad message from %s: type %u (%d expected)," + " magic %u (%d expected).\n", + libcfs_id2str(rpc->crpc_dest), + reply->msg_type, type, + reply->msg_magic, SRPC_MSG_MAGIC); + rc = -EBADMSG; + break; + } + + if (do_bulk && reply->msg_body.reply.status != 0) { + CWARN ("Remote error %d at %s, unlink bulk buffer in " + "case peer didn't initiate bulk transfer\n", + reply->msg_body.reply.status, + libcfs_id2str(rpc->crpc_dest)); + LNetMDUnlink(rpc->crpc_bulk.bk_mdh); + } + + wi->wi_state = SWI_STATE_REPLY_RECEIVED; + } + case SWI_STATE_REPLY_RECEIVED: + if (do_bulk && !rpc->crpc_bulkev.ev_fired) break; + + rc = do_bulk ? rpc->crpc_bulkev.ev_status : 0; + + /* Bulk buffer was unlinked due to remote error. Clear error + * since reply buffer still contains valid data. + * NB rpc->crpc_done shouldn't look into bulk data in case of + * remote error. */ + if (do_bulk && rpc->crpc_bulkev.ev_lnet == LNET_EVENT_UNLINK && + rpc->crpc_status == 0 && reply->msg_body.reply.status != 0) + rc = 0; + + wi->wi_state = SWI_STATE_DONE; + srpc_client_rpc_done(rpc, rc); + return 1; + } + + if (rc != 0) { + spin_lock(&rpc->crpc_lock); + srpc_abort_rpc(rpc, rc); + spin_unlock(&rpc->crpc_lock); + } + +abort: + if (rpc->crpc_aborted) { + LNetMDUnlink(rpc->crpc_reqstmdh); + LNetMDUnlink(rpc->crpc_replymdh); + LNetMDUnlink(rpc->crpc_bulk.bk_mdh); + + if (!srpc_event_pending(rpc)) { + srpc_client_rpc_done(rpc, -EINTR); + return 1; + } + } + return 0; +} + +srpc_client_rpc_t * +srpc_create_client_rpc (lnet_process_id_t peer, int service, + int nbulkiov, int bulklen, + void (*rpc_done)(srpc_client_rpc_t *), + void (*rpc_fini)(srpc_client_rpc_t *), void *priv) +{ + srpc_client_rpc_t *rpc; + + LIBCFS_ALLOC(rpc, offsetof(srpc_client_rpc_t, + crpc_bulk.bk_iovs[nbulkiov])); + if (rpc == NULL) + return NULL; + + srpc_init_client_rpc(rpc, peer, service, nbulkiov, + bulklen, rpc_done, rpc_fini, priv); + return rpc; +} + +/* called with rpc->crpc_lock held */ +static inline void +srpc_queue_rpc (srpc_peer_t *peer, srpc_client_rpc_t *rpc) +{ + int service = rpc->crpc_service; + + LASSERT (peer->stp_nid == rpc->crpc_dest.nid); + LASSERT (srpc_data.rpc_state == SRPC_STATE_RUNNING); + + rpc->crpc_peer = peer; + + spin_lock(&peer->stp_lock); + + /* Framework RPCs that alter session state shall take precedence + * over test RPCs and framework query RPCs */ + if (service <= SRPC_FRAMEWORK_SERVICE_MAX_ID && + service != SRPC_SERVICE_DEBUG && + service != SRPC_SERVICE_QUERY_STAT) + list_add_tail(&rpc->crpc_privl, &peer->stp_ctl_rpcq); + else + list_add_tail(&rpc->crpc_privl, &peer->stp_rpcq); + + srpc_client_rpc_addref(rpc); /* ++ref for peer->*rpcq */ + spin_unlock(&peer->stp_lock); + return; +} + +/* called with rpc->crpc_lock held */ +void +srpc_abort_rpc (srpc_client_rpc_t *rpc, int why) +{ + srpc_peer_t *peer = rpc->crpc_peer; + + LASSERT (why != 0); + + if (rpc->crpc_aborted || /* already aborted */ + rpc->crpc_closed) /* callback imminent */ + return; + + CDEBUG (D_NET, + "Aborting RPC: service %d, peer %s, state %s, why %d\n", + rpc->crpc_service, libcfs_id2str(rpc->crpc_dest), + swi_state2str(rpc->crpc_wi.wi_state), why); + + rpc->crpc_aborted = 1; + rpc->crpc_status = why; + + if (peer != NULL) { + spin_lock(&peer->stp_lock); + + if (!list_empty(&rpc->crpc_privl)) { /* still queued */ + list_del_init(&rpc->crpc_privl); + srpc_client_rpc_decref(rpc); /* --ref for peer->*rpcq */ + rpc->crpc_peer = NULL; /* no credit taken */ + } + + spin_unlock(&peer->stp_lock); + } + + swi_schedule_workitem(&rpc->crpc_wi); + return; +} + +/* called with rpc->crpc_lock held */ +void +srpc_post_rpc (srpc_client_rpc_t *rpc) +{ + srpc_peer_t *peer; + + LASSERT (!rpc->crpc_aborted); + LASSERT (rpc->crpc_peer == NULL); + LASSERT (srpc_data.rpc_state == SRPC_STATE_RUNNING); + LASSERT ((rpc->crpc_bulk.bk_len & ~CFS_PAGE_MASK) == 0); + + CDEBUG (D_NET, "Posting RPC: peer %s, service %d, timeout %d\n", + libcfs_id2str(rpc->crpc_dest), rpc->crpc_service, + rpc->crpc_timeout); + + srpc_add_client_rpc_timer(rpc); + + peer = srpc_nid2peer(rpc->crpc_dest.nid); + if (peer == NULL) { + srpc_abort_rpc(rpc, -ENOMEM); + return; + } + + srpc_queue_rpc(peer, rpc); + + spin_unlock(&rpc->crpc_lock); + srpc_check_sends(peer, 0); + spin_lock(&rpc->crpc_lock); + return; +} + + +int +srpc_send_reply (srpc_server_rpc_t *rpc) +{ + srpc_event_t *ev = &rpc->srpc_ev; + srpc_msg_t *msg = &rpc->srpc_replymsg; + srpc_buffer_t *buffer = rpc->srpc_reqstbuf; + srpc_service_t *sv = rpc->srpc_service; + __u64 rpyid; + int rc; + + LASSERT (buffer != NULL); + rpyid = buffer->buf_msg.msg_body.reqst.rpyid; + + spin_lock(&sv->sv_lock); + + if (!sv->sv_shuttingdown && + sv->sv_id > SRPC_FRAMEWORK_SERVICE_MAX_ID) { + /* Repost buffer before replying since test client + * might send me another RPC once it gets the reply */ + if (srpc_service_post_buffer(sv, buffer) != 0) + CWARN ("Failed to repost %s buffer\n", sv->sv_name); + rpc->srpc_reqstbuf = NULL; + } + + spin_unlock(&sv->sv_lock); + + ev->ev_fired = 0; + ev->ev_data = rpc; + ev->ev_type = SRPC_REPLY_SENT; + + msg->msg_magic = SRPC_MSG_MAGIC; + msg->msg_version = SRPC_MSG_VERSION; + msg->msg_type = srpc_service2reply(sv->sv_id); + + rc = srpc_post_active_rdma(SRPC_RDMA_PORTAL, rpyid, msg, + sizeof(*msg), LNET_MD_OP_PUT, + rpc->srpc_peer, rpc->srpc_self, + &rpc->srpc_replymdh, ev); + if (rc != 0) + ev->ev_fired = 1; /* no more event expected */ + return rc; +} + +/* always called with LNET_LOCK() held, and in thread context */ +void +srpc_lnet_ev_handler (lnet_event_t *ev) +{ + srpc_event_t *rpcev = ev->md.user_ptr; + srpc_client_rpc_t *crpc; + srpc_server_rpc_t *srpc; + srpc_buffer_t *buffer; + srpc_service_t *sv; + srpc_msg_t *msg; + srpc_msg_type_t type; + + LASSERT (!in_interrupt()); + + if (ev->status != 0) { + spin_lock(&srpc_data.rpc_glock); + srpc_data.rpc_counters.errors++; + spin_unlock(&srpc_data.rpc_glock); + } + + rpcev->ev_lnet = ev->type; + + switch (rpcev->ev_type) { + default: + LBUG (); + case SRPC_REQUEST_SENT: + if (ev->status == 0 && ev->type != LNET_EVENT_UNLINK) { + spin_lock(&srpc_data.rpc_glock); + srpc_data.rpc_counters.rpcs_sent++; + spin_unlock(&srpc_data.rpc_glock); + } + case SRPC_REPLY_RCVD: + case SRPC_BULK_REQ_RCVD: + crpc = rpcev->ev_data; + + LASSERT (rpcev == &crpc->crpc_reqstev || + rpcev == &crpc->crpc_replyev || + rpcev == &crpc->crpc_bulkev); + + spin_lock(&crpc->crpc_lock); + + LASSERT (rpcev->ev_fired == 0); + rpcev->ev_fired = 1; + rpcev->ev_status = (ev->type == LNET_EVENT_UNLINK) ? + -EINTR : ev->status; + swi_schedule_workitem(&crpc->crpc_wi); + + spin_unlock(&crpc->crpc_lock); + break; + + case SRPC_REQUEST_RCVD: + sv = rpcev->ev_data; + + LASSERT (rpcev == &sv->sv_ev); + + spin_lock(&sv->sv_lock); + + LASSERT (ev->unlinked); + LASSERT (ev->type == LNET_EVENT_PUT || + ev->type == LNET_EVENT_UNLINK); + LASSERT (ev->type != LNET_EVENT_UNLINK || + sv->sv_shuttingdown); + + buffer = container_of(ev->md.start, srpc_buffer_t, buf_msg); + buffer->buf_peer = ev->initiator; + buffer->buf_self = ev->target.nid; + + sv->sv_nposted_msg--; + LASSERT (sv->sv_nposted_msg >= 0); + + if (sv->sv_shuttingdown) { + /* Leave buffer on sv->sv_posted_msgq since + * srpc_finish_service needs to traverse it. */ + spin_unlock(&sv->sv_lock); + break; + } + + list_del(&buffer->buf_list); /* from sv->sv_posted_msgq */ + msg = &buffer->buf_msg; + type = srpc_service2request(sv->sv_id); + + if (ev->status != 0 || ev->mlength != sizeof(*msg) || + (msg->msg_type != type && + msg->msg_type != __swab32(type)) || + (msg->msg_magic != SRPC_MSG_MAGIC && + msg->msg_magic != __swab32(SRPC_MSG_MAGIC))) { + CERROR ("Dropping RPC (%s) from %s: " + "status %d mlength %d type %u magic %u.\n", + sv->sv_name, libcfs_id2str(ev->initiator), + ev->status, ev->mlength, + msg->msg_type, msg->msg_magic); + + /* NB might drop sv_lock in srpc_service_recycle_buffer, + * sv_nposted_msg++ as an implicit reference to prevent + * sv from disappearing under me */ + sv->sv_nposted_msg++; + srpc_service_recycle_buffer(sv, buffer); + sv->sv_nposted_msg--; + spin_unlock(&sv->sv_lock); + + if (ev->status == 0) { /* status!=0 counted already */ + spin_lock(&srpc_data.rpc_glock); + srpc_data.rpc_counters.errors++; + spin_unlock(&srpc_data.rpc_glock); + } + break; + } + + if (!list_empty(&sv->sv_free_rpcq)) { + srpc = list_entry(sv->sv_free_rpcq.next, + srpc_server_rpc_t, srpc_list); + list_del(&srpc->srpc_list); + + srpc_init_server_rpc(srpc, sv, buffer); + list_add_tail(&srpc->srpc_list, &sv->sv_active_rpcq); + srpc_schedule_server_rpc(srpc); + } else { + list_add_tail(&buffer->buf_list, &sv->sv_blocked_msgq); + } + + spin_unlock(&sv->sv_lock); + + spin_lock(&srpc_data.rpc_glock); + srpc_data.rpc_counters.rpcs_rcvd++; + spin_unlock(&srpc_data.rpc_glock); + break; + + case SRPC_BULK_GET_RPLD: + LASSERT (ev->type == LNET_EVENT_SEND || + ev->type == LNET_EVENT_REPLY || + ev->type == LNET_EVENT_UNLINK); + + if (ev->type == LNET_EVENT_SEND && + ev->status == 0 && !ev->unlinked) + break; /* wait for the final LNET_EVENT_REPLY */ + + case SRPC_BULK_PUT_SENT: + if (ev->status == 0 && ev->type != LNET_EVENT_UNLINK) { + spin_lock(&srpc_data.rpc_glock); + + if (rpcev->ev_type == SRPC_BULK_GET_RPLD) + srpc_data.rpc_counters.bulk_get += ev->mlength; + else + srpc_data.rpc_counters.bulk_put += ev->mlength; + + spin_unlock(&srpc_data.rpc_glock); + } + case SRPC_REPLY_SENT: + srpc = rpcev->ev_data; + sv = srpc->srpc_service; + + LASSERT (rpcev == &srpc->srpc_ev); + + spin_lock(&sv->sv_lock); + rpcev->ev_fired = 1; + rpcev->ev_status = (ev->type == LNET_EVENT_UNLINK) ? + -EINTR : ev->status; + srpc_schedule_server_rpc(srpc); + spin_unlock(&sv->sv_lock); + break; + } + + return; +} + +#ifndef __KERNEL__ + +int +srpc_check_event (int timeout) +{ + lnet_event_t ev; + int rc; + int i; + + rc = LNetEQPoll(&srpc_data.rpc_lnet_eq, 1, + timeout * 1000, &ev, &i); + if (rc == 0) + return 0; + + LASSERT (rc == -EOVERFLOW || rc == 1); + + /* We can't affort to miss any events... */ + if (rc == -EOVERFLOW) { + CERROR ("Dropped an event!!!\n"); + abort(); + } + + srpc_lnet_ev_handler(&ev); + return 1; +} + +#endif + +int +srpc_startup (void) +{ + int i; + int rc; + + memset(&srpc_data, 0, sizeof(struct smoketest_rpc)); + spin_lock_init(&srpc_data.rpc_glock); + + /* 1 second pause to avoid timestamp reuse */ + cfs_pause(cfs_time_seconds(1)); + srpc_data.rpc_matchbits = ((__u64) cfs_time_current_sec()) << 48; + + srpc_data.rpc_state = SRPC_STATE_NONE; + + LIBCFS_ALLOC(srpc_data.rpc_peers, + sizeof(struct list_head) * SRPC_PEER_HASH_SIZE); + if (srpc_data.rpc_peers == NULL) { + CERROR ("Failed to alloc peer hash.\n"); + return -ENOMEM; + } + + for (i = 0; i < SRPC_PEER_HASH_SIZE; i++) + CFS_INIT_LIST_HEAD(&srpc_data.rpc_peers[i]); + +#ifdef __KERNEL__ + rc = LNetNIInit(LUSTRE_SRV_LNET_PID); +#else + rc = LNetNIInit(getpid()); +#endif + if (rc < 0) { + CERROR ("LNetNIInit() has failed: %d\n", rc); + LIBCFS_FREE(srpc_data.rpc_peers, + sizeof(struct list_head) * SRPC_PEER_HASH_SIZE); + return rc; + } + + srpc_data.rpc_state = SRPC_STATE_NI_INIT; + + srpc_data.rpc_lnet_eq = LNET_EQ_NONE; +#ifdef __KERNEL__ + rc = LNetEQAlloc(16, srpc_lnet_ev_handler, &srpc_data.rpc_lnet_eq); +#else + rc = LNetEQAlloc(10240, LNET_EQ_HANDLER_NONE, &srpc_data.rpc_lnet_eq); +#endif + if (rc != 0) { + CERROR("LNetEQAlloc() has failed: %d\n", rc); + goto bail; + } + + rc = LNetSetLazyPortal(SRPC_FRAMEWORK_REQUEST_PORTAL); + LASSERT (rc == 0); + + srpc_data.rpc_state = SRPC_STATE_EQ_INIT; + + rc = swi_startup(); + if (rc != 0) + goto bail; + + srpc_data.rpc_state = SRPC_STATE_WI_INIT; + + rc = stt_startup(); + +bail: + if (rc != 0) + srpc_shutdown(); + else + srpc_data.rpc_state = SRPC_STATE_RUNNING; + + return rc; +} + +void +srpc_shutdown (void) +{ + int i; + int rc; + int state; + + state = srpc_data.rpc_state; + srpc_data.rpc_state = SRPC_STATE_STOPPING; + + switch (state) { + default: + LBUG (); + case SRPC_STATE_RUNNING: + spin_lock(&srpc_data.rpc_glock); + + for (i = 0; i <= SRPC_SERVICE_MAX_ID; i++) { + srpc_service_t *sv = srpc_data.rpc_services[i]; + + LASSERTF (sv == NULL, + "service not empty: id %d, name %s\n", + i, sv->sv_name); + } + + spin_unlock(&srpc_data.rpc_glock); + + stt_shutdown(); + + case SRPC_STATE_WI_INIT: + swi_shutdown(); + + case SRPC_STATE_EQ_INIT: + rc = LNetClearLazyPortal(SRPC_FRAMEWORK_REQUEST_PORTAL); + LASSERT (rc == 0); + rc = LNetEQFree(srpc_data.rpc_lnet_eq); + LASSERT (rc == 0); /* the EQ should have no user by now */ + + case SRPC_STATE_NI_INIT: + LNetNIFini(); + break; + } + + /* srpc_peer_t's are kept in hash until shutdown */ + for (i = 0; i < SRPC_PEER_HASH_SIZE; i++) { + srpc_peer_t *peer; + + while (!list_empty(&srpc_data.rpc_peers[i])) { + peer = list_entry(srpc_data.rpc_peers[i].next, + srpc_peer_t, stp_list); + list_del(&peer->stp_list); + + LASSERT (list_empty(&peer->stp_rpcq)); + LASSERT (list_empty(&peer->stp_ctl_rpcq)); + LASSERT (peer->stp_credits == SRPC_PEER_CREDITS); + + LIBCFS_FREE(peer, sizeof(srpc_peer_t)); + } + } + + LIBCFS_FREE(srpc_data.rpc_peers, + sizeof(struct list_head) * SRPC_PEER_HASH_SIZE); + return; +} diff --git a/lnet/selftest/rpc.h b/lnet/selftest/rpc.h new file mode 100644 index 0000000..6acd80f --- /dev/null +++ b/lnet/selftest/rpc.h @@ -0,0 +1,235 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + */ +#ifndef __SELFTEST_RPC_H__ +#define __SELFTEST_RPC_H__ + +#include + +/* + * LST wired structures + * + * XXX: *REPLY == *REQST + 1 + */ +typedef enum { + SRPC_MSG_MKSN_REQST = 0, + SRPC_MSG_MKSN_REPLY = 1, + SRPC_MSG_RMSN_REQST = 2, + SRPC_MSG_RMSN_REPLY = 3, + SRPC_MSG_BATCH_REQST = 4, + SRPC_MSG_BATCH_REPLY = 5, + SRPC_MSG_STAT_REQST = 6, + SRPC_MSG_STAT_REPLY = 7, + SRPC_MSG_TEST_REQST = 8, + SRPC_MSG_TEST_REPLY = 9, + SRPC_MSG_DEBUG_REQST = 10, + SRPC_MSG_DEBUG_REPLY = 11, + SRPC_MSG_BRW_REQST = 12, + SRPC_MSG_BRW_REPLY = 13, + SRPC_MSG_PING_REQST = 14, + SRPC_MSG_PING_REPLY = 15, + SRPC_MSG_JOIN_REQST = 16, + SRPC_MSG_JOIN_REPLY = 17, +} srpc_msg_type_t; + +/* CAVEAT EMPTOR: + * All srpc_*_reqst_t's 1st field must be matchbits of reply buffer, + * and 2nd field matchbits of bulk buffer if any. + * + * All srpc_*_reply_t's 1st field must be a __u32 status, and 2nd field + * session id if needed. + */ +typedef struct { + __u64 rpyid; /* reply buffer matchbits */ + __u64 bulkid; /* bulk buffer matchbits */ +} WIRE_ATTR srpc_generic_reqst_t; + +typedef struct { + __u32 status; + lst_sid_t sid; +} WIRE_ATTR srpc_generic_reply_t; + +/* FRAMEWORK RPCs */ +typedef struct { + __u64 mksn_rpyid; /* reply buffer matchbits */ + lst_sid_t mksn_sid; /* session id */ + __u32 mksn_force; /* use brute force */ + char mksn_name[LST_NAME_SIZE]; +} WIRE_ATTR srpc_mksn_reqst_t; /* make session request */ + +typedef struct { + __u32 mksn_status; /* session status */ + lst_sid_t mksn_sid; /* session id */ + __u32 mksn_timeout; /* session timeout */ + char mksn_name[LST_NAME_SIZE]; +} WIRE_ATTR srpc_mksn_reply_t; /* make session reply */ + +typedef struct { + __u64 rmsn_rpyid; /* reply buffer matchbits */ + lst_sid_t rmsn_sid; /* session id */ +} WIRE_ATTR srpc_rmsn_reqst_t; /* remove session request */ + +typedef struct { + __u32 rmsn_status; + lst_sid_t rmsn_sid; /* session id */ +} WIRE_ATTR srpc_rmsn_reply_t; /* remove session reply */ + +typedef struct { + __u64 join_rpyid; /* reply buffer matchbits */ + lst_sid_t join_sid; /* session id to join */ + char join_group[LST_NAME_SIZE]; /* group name */ +} WIRE_ATTR srpc_join_reqst_t; + +typedef struct { + __u32 join_status; /* returned status */ + lst_sid_t join_sid; /* session id */ + __u32 join_timeout; /* # seconds' inactivity to expire */ + char join_session[LST_NAME_SIZE]; /* session name */ +} WIRE_ATTR srpc_join_reply_t; + +typedef struct { + __u64 dbg_rpyid; /* reply buffer matchbits */ + lst_sid_t dbg_sid; /* session id */ + __u32 dbg_flags; /* bitmap of debug */ +} WIRE_ATTR srpc_debug_reqst_t; + +typedef struct { + __u32 dbg_status; /* returned code */ + lst_sid_t dbg_sid; /* session id */ + __u32 dbg_timeout; /* session timeout */ + __u32 dbg_nbatch; /* # of batches in the node */ + char dbg_name[LST_NAME_SIZE]; /* session name */ +} WIRE_ATTR srpc_debug_reply_t; + +#define SRPC_BATCH_OPC_RUN 1 +#define SRPC_BATCH_OPC_STOP 2 +#define SRPC_BATCH_OPC_QUERY 3 + +typedef struct { + __u64 bar_rpyid; /* reply buffer matchbits */ + lst_sid_t bar_sid; /* session id */ + lst_bid_t bar_bid; /* batch id */ + __u32 bar_opc; /* create/start/stop batch */ + __u32 bar_testidx; /* index of test */ + __u32 bar_arg; /* parameters */ +} WIRE_ATTR srpc_batch_reqst_t; + +typedef struct { + __u32 bar_status; /* status of request */ + lst_sid_t bar_sid; /* session id */ + __u32 bar_active; /* # of active tests in batch/test */ + __u32 bar_time; /* remained time */ +} WIRE_ATTR srpc_batch_reply_t; + +typedef struct { + __u64 str_rpyid; /* reply buffer matchbits */ + lst_sid_t str_sid; /* session id */ + __u32 str_type; /* type of stat */ +} WIRE_ATTR srpc_stat_reqst_t; + +typedef struct { + __u32 str_status; + lst_sid_t str_sid; + sfw_counters_t str_fw; + srpc_counters_t str_rpc; + lnet_counters_t str_lnet; +} WIRE_ATTR srpc_stat_reply_t; + +typedef struct { + __u32 blk_opc; /* bulk operation code */ + __u32 blk_npg; /* # pages */ + __u32 blk_flags; /* reserved flags */ +} WIRE_ATTR test_bulk_req_t; + +typedef struct { + __u32 png_size; /* size of ping message */ + __u32 png_flags; /* reserved flags */ +} WIRE_ATTR test_ping_req_t; + +typedef struct { + __u64 tsr_rpyid; /* reply buffer matchbits */ + __u64 tsr_bulkid; /* bulk buffer matchbits */ + lst_sid_t tsr_sid; /* session id */ + lst_bid_t tsr_bid; /* batch id */ + __u32 tsr_service; /* test type: bulk|ping|... */ + /* test client loop count or # server buffers needed */ + __u32 tsr_loop; + __u32 tsr_concur; /* concurrency of test */ + __u8 tsr_is_client; /* is test client or not */ + __u8 tsr_stop_onerr; /* stop on error */ + __u32 tsr_ndest; /* # of dest nodes */ + + union { + test_bulk_req_t bulk; + test_ping_req_t ping; + } tsr_u; +} WIRE_ATTR srpc_test_reqst_t; + +typedef struct { + __u32 tsr_status; /* returned code */ + lst_sid_t tsr_sid; +} WIRE_ATTR srpc_test_reply_t; + +/* TEST RPCs */ +typedef struct { + __u64 pnr_rpyid; + __u32 pnr_magic; + __u32 pnr_seq; + __u64 pnr_time_sec; + __u64 pnr_time_usec; +} WIRE_ATTR srpc_ping_reqst_t; + +typedef struct { + __u32 pnr_status; + __u32 pnr_magic; + __u32 pnr_seq; +} WIRE_ATTR srpc_ping_reply_t; + +typedef struct { + __u64 brw_rpyid; /* reply buffer matchbits */ + __u64 brw_bulkid; /* bulk buffer matchbits */ + __u32 brw_rw; /* read or write */ + __u32 brw_len; /* bulk data len */ + __u32 brw_flags; /* bulk data patterns */ +} WIRE_ATTR srpc_brw_reqst_t; /* bulk r/w request */ + +typedef struct { + __u32 brw_status; +} WIRE_ATTR srpc_brw_reply_t; /* bulk r/w reply */ + +#define SRPC_MSG_MAGIC 0xeeb0f00d +#define SRPC_MSG_VERSION 1 +typedef struct { + __u32 msg_magic; /* magic */ + __u32 msg_version; /* # version */ + __u32 msg_type; /* what's in msg_body? srpc_msg_type_t */ + __u32 msg_reserved0; /* reserved seats */ + __u32 msg_reserved1; + __u32 msg_reserved2; + union { + srpc_generic_reqst_t reqst; + srpc_generic_reply_t reply; + + srpc_mksn_reqst_t mksn_reqst; + srpc_mksn_reply_t mksn_reply; + srpc_rmsn_reqst_t rmsn_reqst; + srpc_rmsn_reply_t rmsn_reply; + srpc_debug_reqst_t dbg_reqst; + srpc_debug_reply_t dbg_reply; + srpc_batch_reqst_t bat_reqst; + srpc_batch_reply_t bat_reply; + srpc_stat_reqst_t stat_reqst; + srpc_stat_reply_t stat_reply; + srpc_test_reqst_t tes_reqst; + srpc_test_reply_t tes_reply; + srpc_join_reqst_t join_reqst; + srpc_join_reply_t join_reply; + + srpc_ping_reqst_t ping_reqst; + srpc_ping_reply_t ping_reply; + srpc_brw_reqst_t brw_reqst; + srpc_brw_reply_t brw_reply; + } msg_body; +} WIRE_ATTR srpc_msg_t; + +#endif /* __SELFTEST_RPC_H__ */ diff --git a/lnet/selftest/selftest.h b/lnet/selftest/selftest.h new file mode 100644 index 0000000..72cc17a --- /dev/null +++ b/lnet/selftest/selftest.h @@ -0,0 +1,568 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2001, 2002 Cluster File Systems, Inc. + * Author: Isaac Huang + * + */ +#ifndef __SELFTEST_SELFTEST_H__ +#define __SELFTEST_SELFTEST_H__ + +#define LNET_ONLY + +#include +#include +#include +#include +#include + +#ifndef __KERNEL__ +#include /* userland spinlock_t and atomic_t */ +#endif + +#include "rpc.h" +#include "timer.h" + +#ifndef MADE_WITHOUT_COMPROMISE +#define MADE_WITHOUT_COMPROMISE +#endif + + +#define SWI_STATE_NEWBORN 0 +#define SWI_STATE_REPLY_SUBMITTED 1 +#define SWI_STATE_REPLY_SENT 2 +#define SWI_STATE_REQUEST_SUBMITTED 3 +#define SWI_STATE_REQUEST_SENT 4 +#define SWI_STATE_REPLY_RECEIVED 5 +#define SWI_STATE_BULK_STARTED 6 +#define SWI_STATE_DONE 10 + +/* forward refs */ +struct swi_workitem; +struct srpc_service; +struct sfw_test_unit; +struct sfw_test_instance; + +/* + * A workitems is deferred work with these semantics: + * - a workitem always runs in thread context. + * - a workitem can be concurrent with other workitems but is strictly + * serialized with respect to itself. + * - no CPU affinity, a workitem does not necessarily run on the same CPU + * that schedules it. However, this might change in the future. + * - if a workitem is scheduled again before it has a chance to run, it + * runs only once. + * - if a workitem is scheduled while it runs, it runs again after it + * completes; this ensures that events occurring while other events are + * being processed receive due attention. This behavior also allows a + * workitem to reschedule itself. + * + * Usage notes: + * - a workitem can sleep but it should be aware of how that sleep might + * affect others. + * - a workitem runs inside a kernel thread so there's no user space to access. + * - do not use a workitem if the scheduling latency can't be tolerated. + * + * When wi_action returns non-zero, it means the workitem has either been + * freed or reused and workitem scheduler won't touch it any more. + */ +typedef int (*swi_action_t) (struct swi_workitem *); +typedef struct swi_workitem { + struct list_head wi_list; /* chain on runq */ + int wi_state; + swi_action_t wi_action; + void *wi_data; + unsigned int wi_running:1; + unsigned int wi_scheduled:1; +} swi_workitem_t; + +static inline void +swi_init_workitem (swi_workitem_t *wi, void *data, swi_action_t action) +{ + CFS_INIT_LIST_HEAD(&wi->wi_list); + + wi->wi_running = 0; + wi->wi_scheduled = 0; + wi->wi_data = data; + wi->wi_action = action; + wi->wi_state = SWI_STATE_NEWBORN; +} + +#define SWI_RESCHED 128 /* # workitem scheduler loops before reschedule */ + +/* services below SRPC_FRAMEWORK_SERVICE_MAX_ID are framework + * services, e.g. create/modify session. + */ +#define SRPC_SERVICE_DEBUG 0 +#define SRPC_SERVICE_MAKE_SESSION 1 +#define SRPC_SERVICE_REMOVE_SESSION 2 +#define SRPC_SERVICE_BATCH 3 +#define SRPC_SERVICE_TEST 4 +#define SRPC_SERVICE_QUERY_STAT 5 +#define SRPC_SERVICE_JOIN 6 +#define SRPC_FRAMEWORK_SERVICE_MAX_ID 10 +/* other services start from SRPC_FRAMEWORK_SERVICE_MAX_ID+1 */ +#define SRPC_SERVICE_BRW 11 +#define SRPC_SERVICE_PING 12 +#define SRPC_SERVICE_MAX_ID 12 + +#define SRPC_REQUEST_PORTAL 50 +/* a lazy portal for framework RPC requests */ +#define SRPC_FRAMEWORK_REQUEST_PORTAL 51 +/* all reply/bulk RDMAs go to this portal */ +#define SRPC_RDMA_PORTAL 52 + +static inline srpc_msg_type_t +srpc_service2request (int service) +{ + switch (service) { + default: + LBUG (); + case SRPC_SERVICE_DEBUG: + return SRPC_MSG_DEBUG_REQST; + + case SRPC_SERVICE_MAKE_SESSION: + return SRPC_MSG_MKSN_REQST; + + case SRPC_SERVICE_REMOVE_SESSION: + return SRPC_MSG_RMSN_REQST; + + case SRPC_SERVICE_BATCH: + return SRPC_MSG_BATCH_REQST; + + case SRPC_SERVICE_TEST: + return SRPC_MSG_TEST_REQST; + + case SRPC_SERVICE_QUERY_STAT: + return SRPC_MSG_STAT_REQST; + + case SRPC_SERVICE_BRW: + return SRPC_MSG_BRW_REQST; + + case SRPC_SERVICE_PING: + return SRPC_MSG_PING_REQST; + + case SRPC_SERVICE_JOIN: + return SRPC_MSG_JOIN_REQST; + } +} + +static inline srpc_msg_type_t +srpc_service2reply (int service) +{ + return srpc_service2request(service) + 1; +} + +typedef enum { + SRPC_BULK_REQ_RCVD = 0, /* passive bulk request(PUT sink/GET source) received */ + SRPC_BULK_PUT_SENT = 1, /* active bulk PUT sent (source) */ + SRPC_BULK_GET_RPLD = 2, /* active bulk GET replied (sink) */ + SRPC_REPLY_RCVD = 3, /* incoming reply received */ + SRPC_REPLY_SENT = 4, /* outgoing reply sent */ + SRPC_REQUEST_RCVD = 5, /* incoming request received */ + SRPC_REQUEST_SENT = 6, /* outgoing request sent */ +} srpc_event_type_t; + +/* RPC event */ +typedef struct { + srpc_event_type_t ev_type; /* what's up */ + lnet_event_kind_t ev_lnet; /* LNet event type */ + int ev_fired; /* LNet event fired? */ + int ev_status; /* LNet event status */ + void *ev_data; /* owning server/client RPC */ +} srpc_event_t; + +typedef struct { + int bk_len; /* len of bulk data */ + lnet_handle_md_t bk_mdh; + int bk_sink; /* sink/source */ + int bk_niov; /* # iov in bk_iovs */ +#ifdef __KERNEL__ + lnet_kiov_t bk_iovs[0]; +#else + cfs_page_t **bk_pages; + lnet_md_iovec_t bk_iovs[0]; +#endif +} srpc_bulk_t; /* bulk descriptor */ + +typedef struct srpc_peer { + struct list_head stp_list; /* chain on peer hash */ + struct list_head stp_rpcq; /* q of non-control RPCs */ + struct list_head stp_ctl_rpcq; /* q of control RPCs */ + spinlock_t stp_lock; /* serialize */ + lnet_nid_t stp_nid; + int stp_credits; /* available credits */ +} srpc_peer_t; + +/* message buffer descriptor */ +typedef struct { + struct list_head buf_list; /* chain on srpc_service::*_msgq */ + srpc_msg_t buf_msg; + lnet_handle_md_t buf_mdh; + lnet_nid_t buf_self; + lnet_process_id_t buf_peer; +} srpc_buffer_t; + +/* server-side state of a RPC */ +typedef struct srpc_server_rpc { + struct list_head srpc_list; /* chain on srpc_service::*_rpcq */ + struct srpc_service *srpc_service; + swi_workitem_t srpc_wi; + srpc_event_t srpc_ev; /* bulk/reply event */ + lnet_nid_t srpc_self; + lnet_process_id_t srpc_peer; + srpc_msg_t srpc_replymsg; + lnet_handle_md_t srpc_replymdh; + srpc_buffer_t *srpc_reqstbuf; + srpc_bulk_t *srpc_bulk; + + int srpc_status; + void (*srpc_done)(struct srpc_server_rpc *); +} srpc_server_rpc_t; + +/* client-side state of a RPC */ +typedef struct srpc_client_rpc { + struct list_head crpc_list; /* chain on user's lists */ + struct list_head crpc_privl; /* chain on srpc_peer_t::*rpcq */ + spinlock_t crpc_lock; /* serialize */ + int crpc_service; + atomic_t crpc_refcount; + int crpc_timeout; /* # seconds to wait for reply */ + stt_timer_t crpc_timer; + swi_workitem_t crpc_wi; + lnet_process_id_t crpc_dest; + srpc_peer_t *crpc_peer; + + void (*crpc_done)(struct srpc_client_rpc *); + void (*crpc_fini)(struct srpc_client_rpc *); + int crpc_status; /* completion status */ + void *crpc_priv; /* caller data */ + + /* state flags */ + unsigned int crpc_aborted:1; /* being given up */ + unsigned int crpc_closed:1; /* completed */ + + /* RPC events */ + srpc_event_t crpc_bulkev; /* bulk event */ + srpc_event_t crpc_reqstev; /* request event */ + srpc_event_t crpc_replyev; /* reply event */ + + /* bulk, request(reqst), and reply exchanged on wire */ + srpc_msg_t crpc_reqstmsg; + srpc_msg_t crpc_replymsg; + lnet_handle_md_t crpc_reqstmdh; + lnet_handle_md_t crpc_replymdh; + srpc_bulk_t crpc_bulk; +} srpc_client_rpc_t; + +#define srpc_client_rpc_size(rpc) \ +offsetof(srpc_client_rpc_t, crpc_bulk.bk_iovs[(rpc)->crpc_bulk.bk_niov]) + +#define srpc_client_rpc_addref(rpc) \ +do { \ + CDEBUG(D_NET, "RPC[%p] -> %s (%d)++\n", \ + (rpc), libcfs_id2str((rpc)->crpc_dest), \ + atomic_read(&(rpc)->crpc_refcount)); \ + LASSERT(atomic_read(&(rpc)->crpc_refcount) > 0); \ + atomic_inc(&(rpc)->crpc_refcount); \ +} while (0) + +#define srpc_client_rpc_decref(rpc) \ +do { \ + CDEBUG(D_NET, "RPC[%p] -> %s (%d)--\n", \ + (rpc), libcfs_id2str((rpc)->crpc_dest), \ + atomic_read(&(rpc)->crpc_refcount)); \ + LASSERT(atomic_read(&(rpc)->crpc_refcount) > 0); \ + if (atomic_dec_and_test(&(rpc)->crpc_refcount)) \ + srpc_destroy_client_rpc(rpc); \ +} while (0) + +#define srpc_event_pending(rpc) ((rpc)->crpc_bulkev.ev_fired == 0 || \ + (rpc)->crpc_reqstev.ev_fired == 0 || \ + (rpc)->crpc_replyev.ev_fired == 0) + +typedef struct srpc_service { + int sv_id; /* service id */ + const char *sv_name; /* human readable name */ + int sv_nprune; /* # posted RPC to be pruned */ + int sv_concur; /* max # concurrent RPCs */ + + spinlock_t sv_lock; + int sv_shuttingdown; + srpc_event_t sv_ev; /* LNet event */ + int sv_nposted_msg; /* # posted message buffers */ + struct list_head sv_free_rpcq; /* free RPC descriptors */ + struct list_head sv_active_rpcq; /* in-flight RPCs */ + struct list_head sv_posted_msgq; /* posted message buffers */ + struct list_head sv_blocked_msgq; /* blocked for RPC descriptor */ + + /* Service callbacks: + * - sv_handler: process incoming RPC request + * - sv_bulk_ready: notify bulk data + */ + int (*sv_handler) (srpc_server_rpc_t *); + int (*sv_bulk_ready) (srpc_server_rpc_t *, int); +} srpc_service_t; + +#define SFW_POST_BUFFERS 8 +#define SFW_SERVICE_CONCURRENCY (SFW_POST_BUFFERS/2) + +typedef struct { + struct list_head sn_list; /* chain on fw_zombie_sessions */ + lst_sid_t sn_id; /* unique identifier */ + unsigned int sn_timeout; /* # seconds' inactivity to expire */ + int sn_timer_active; + stt_timer_t sn_timer; + struct list_head sn_batches; /* list of batches */ + char sn_name[LST_NAME_SIZE]; + atomic_t sn_brw_errors; +} sfw_session_t; + +#define sfw_sid_equal(sid0, sid1) ((sid0).ses_nid == (sid1).ses_nid && \ + (sid0).ses_stamp == (sid1).ses_stamp) + +typedef struct { + struct list_head bat_list; /* chain on sn_batches */ + lst_bid_t bat_id; /* batch id */ + int bat_error; /* error code of batch */ + sfw_session_t *bat_session; /* batch's session */ + atomic_t bat_nactive; /* # of active tests */ + struct list_head bat_tests; /* test instances */ +} sfw_batch_t; + +typedef struct { + int (*tso_init)(struct sfw_test_instance *tsi); /* intialize test client */ + void (*tso_fini)(struct sfw_test_instance *tsi); /* finalize test client */ + int (*tso_prep_rpc)(struct sfw_test_unit *tsu, + lnet_process_id_t dest, + srpc_client_rpc_t **rpc); /* prep a tests rpc */ + void (*tso_done_rpc)(struct sfw_test_unit *tsu, + srpc_client_rpc_t *rpc); /* done a test rpc */ +} sfw_test_client_ops_t; + +typedef struct sfw_test_instance { + struct list_head tsi_list; /* chain on batch */ + int tsi_service; /* test type */ + sfw_batch_t *tsi_batch; /* batch */ + sfw_test_client_ops_t *tsi_ops; /* test client operations */ + + /* public parameter for all test units */ + int tsi_is_client:1; /* is test client */ + int tsi_stop_onerr:1; /* stop on error */ + int tsi_concur; /* concurrency */ + int tsi_loop; /* loop count */ + + /* status of test instance */ + spinlock_t tsi_lock; /* serialize */ + int tsi_stopping:1; /* test is stopping */ + atomic_t tsi_nactive; /* # of active test unit */ + struct list_head tsi_units; /* test units */ + struct list_head tsi_free_rpcs; /* free rpcs */ + struct list_head tsi_active_rpcs; /* active rpcs */ + + union { + test_bulk_req_t bulk; /* bulk parameter */ + test_ping_req_t ping; /* ping parameter */ + } tsi_u; +} sfw_test_instance_t; + +/* XXX: trailing (CFS_PAGE_SIZE % sizeof(lnet_process_id_t)) bytes at + * the end of pages are not used */ +#define SFW_MAX_CONCUR LST_MAX_CONCUR +#define SFW_ID_PER_PAGE (CFS_PAGE_SIZE / sizeof(lnet_process_id_t)) +#define SFW_MAX_NDESTS (LNET_MAX_IOV * SFW_ID_PER_PAGE) +#define sfw_id_pages(n) (((n) + SFW_ID_PER_PAGE - 1) / SFW_ID_PER_PAGE) + +typedef struct sfw_test_unit { + struct list_head tsu_list; /* chain on lst_test_instance */ + lnet_process_id_t tsu_dest; /* id of dest node */ + int tsu_loop; /* loop count of the test */ + int tsu_error; /* error code */ + sfw_test_instance_t *tsu_instance; /* pointer to test instance */ + void *tsu_private; /* private data */ + swi_workitem_t tsu_worker; /* workitem of the test unit */ +} sfw_test_unit_t; + +typedef struct { + struct list_head tsc_list; /* chain on fw_tests */ + srpc_service_t *tsc_srv_service; /* test service */ + sfw_test_client_ops_t *tsc_cli_ops; /* ops of test client */ +} sfw_test_case_t; + + +srpc_client_rpc_t * +sfw_create_rpc(lnet_process_id_t peer, int service, int nbulkiov, int bulklen, + void (*done) (srpc_client_rpc_t *), void *priv); +int sfw_create_test_rpc(sfw_test_unit_t *tsu, lnet_process_id_t peer, + int nblk, int blklen, srpc_client_rpc_t **rpc); +void sfw_abort_rpc(srpc_client_rpc_t *rpc); +void sfw_post_rpc(srpc_client_rpc_t *rpc); +void sfw_client_rpc_done(srpc_client_rpc_t *rpc); +void sfw_unpack_message(srpc_msg_t *msg); +void sfw_free_pages(srpc_server_rpc_t *rpc); +void sfw_add_bulk_page(srpc_bulk_t *bk, cfs_page_t *pg, int i); +int sfw_alloc_pages(srpc_server_rpc_t *rpc, int npages, int sink); + +srpc_client_rpc_t * +srpc_create_client_rpc(lnet_process_id_t peer, int service, + int nbulkiov, int bulklen, + void (*rpc_done)(srpc_client_rpc_t *), + void (*rpc_fini)(srpc_client_rpc_t *), void *priv); +void srpc_post_rpc(srpc_client_rpc_t *rpc); +void srpc_abort_rpc(srpc_client_rpc_t *rpc, int why); +void srpc_free_bulk(srpc_bulk_t *bk); +srpc_bulk_t *srpc_alloc_bulk(int npages, int sink); +int srpc_send_rpc(swi_workitem_t *wi); +int srpc_send_reply(srpc_server_rpc_t *rpc); +int srpc_add_service(srpc_service_t *sv); +int srpc_remove_service(srpc_service_t *sv); +void srpc_shutdown_service(srpc_service_t *sv); +int srpc_finish_service(srpc_service_t *sv); +int srpc_service_add_buffers(srpc_service_t *sv, int nbuffer); +void srpc_service_remove_buffers(srpc_service_t *sv, int nbuffer); +void srpc_get_counters(srpc_counters_t *cnt); +void srpc_set_counters(const srpc_counters_t *cnt); + +void swi_kill_workitem(swi_workitem_t *wi); +void swi_schedule_workitem(swi_workitem_t *wi); +void swi_schedule_serial_workitem(swi_workitem_t *wi); +int swi_startup(void); +int sfw_startup(void); +int srpc_startup(void); +void swi_shutdown(void); +void sfw_shutdown(void); +void srpc_shutdown(void); + +static inline void +srpc_destroy_client_rpc (srpc_client_rpc_t *rpc) +{ + LASSERT (rpc != NULL); + LASSERT (!srpc_event_pending(rpc)); + LASSERT (list_empty(&rpc->crpc_privl)); + LASSERT (atomic_read(&rpc->crpc_refcount) == 0); +#ifndef __KERNEL__ + LASSERT (rpc->crpc_bulk.bk_pages == NULL); +#endif + + if (rpc->crpc_fini == NULL) { + LIBCFS_FREE(rpc, srpc_client_rpc_size(rpc)); + } else { + (*rpc->crpc_fini) (rpc); + } + + return; +} + +static inline void +srpc_init_client_rpc (srpc_client_rpc_t *rpc, lnet_process_id_t peer, + int service, int nbulkiov, int bulklen, + void (*rpc_done)(srpc_client_rpc_t *), + void (*rpc_fini)(srpc_client_rpc_t *), void *priv) +{ + LASSERT (nbulkiov <= LNET_MAX_IOV); + + memset(rpc, 0, offsetof(srpc_client_rpc_t, + crpc_bulk.bk_iovs[nbulkiov])); + + CFS_INIT_LIST_HEAD(&rpc->crpc_list); + CFS_INIT_LIST_HEAD(&rpc->crpc_privl); + swi_init_workitem(&rpc->crpc_wi, rpc, srpc_send_rpc); + spin_lock_init(&rpc->crpc_lock); + atomic_set(&rpc->crpc_refcount, 1); /* 1 ref for caller */ + + rpc->crpc_dest = peer; + rpc->crpc_priv = priv; + rpc->crpc_service = service; + rpc->crpc_bulk.bk_len = bulklen; + rpc->crpc_bulk.bk_niov = nbulkiov; + rpc->crpc_done = rpc_done; + rpc->crpc_fini = rpc_fini; + rpc->crpc_reqstmdh = + rpc->crpc_replymdh = + rpc->crpc_bulk.bk_mdh = LNET_INVALID_HANDLE; + + /* no event is expected at this point */ + rpc->crpc_bulkev.ev_fired = + rpc->crpc_reqstev.ev_fired = + rpc->crpc_replyev.ev_fired = 1; + + rpc->crpc_reqstmsg.msg_magic = SRPC_MSG_MAGIC; + rpc->crpc_reqstmsg.msg_version = SRPC_MSG_VERSION; + rpc->crpc_reqstmsg.msg_type = srpc_service2request(service); + return; +} + +static inline const char * +swi_state2str (int state) +{ +#define STATE2STR(x) case x: return #x + switch(state) { + default: + LBUG(); + STATE2STR(SWI_STATE_NEWBORN); + STATE2STR(SWI_STATE_REPLY_SUBMITTED); + STATE2STR(SWI_STATE_REPLY_SENT); + STATE2STR(SWI_STATE_REQUEST_SUBMITTED); + STATE2STR(SWI_STATE_REQUEST_SENT); + STATE2STR(SWI_STATE_REPLY_RECEIVED); + STATE2STR(SWI_STATE_BULK_STARTED); + STATE2STR(SWI_STATE_DONE); + } +#undef STATE2STR +} + +#define UNUSED(x) ( (void)(x) ) + +#ifndef __KERNEL__ + +int stt_poll_interval(void); + +int stt_check_events(void); +int swi_check_events(void); +int srpc_check_event(int timeout); + +int lnet_selftest_init(void); +void lnet_selftest_fini(void); +int selftest_wait_events(void); + +#else + +#define selftest_wait_events() cfs_pause(cfs_time_seconds(1)) + +#endif + +#define lst_wait_until(cond, lock, fmt, a...) \ +do { \ + int __I = 2; \ + while (!(cond)) { \ + __I++; \ + CDEBUG(((__I & (-__I)) == __I) ? D_WARNING : \ + D_NET, /* 2**n? */ \ + fmt, ## a); \ + spin_unlock(&(lock)); \ + \ + selftest_wait_events(); \ + \ + spin_lock(&(lock)); \ + } \ +} while (0) + +static inline void +srpc_wait_service_shutdown (srpc_service_t *sv) +{ + int i = 2; + + spin_lock(&sv->sv_lock); + LASSERT (sv->sv_shuttingdown); + spin_unlock(&sv->sv_lock); + + while (srpc_finish_service(sv) == 0) { + i++; + CDEBUG (((i & -i) == i) ? D_WARNING : D_NET, + "Waiting for %s service to shutdown...\n", + sv->sv_name); + selftest_wait_events(); + } +} + +#endif /* __SELFTEST_SELFTEST_H__ */ diff --git a/lnet/selftest/timer.c b/lnet/selftest/timer.c new file mode 100644 index 0000000..97d9297 --- /dev/null +++ b/lnet/selftest/timer.c @@ -0,0 +1,247 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2001, 2002 Cluster File Systems, Inc. + * Author: Isaac Huang + * + */ + +#define DEBUG_SUBSYSTEM S_LNET + +#include +#include +#include + +#include "selftest.h" + + +/* + * Timers are implemented as a sorted queue of expiry times. The queue + * is slotted, with each slot holding timers which expire in a + * 2**STTIMER_MINPOLL (8) second period. The timers in each slot are + * sorted by increasing expiry time. The number of slots is 2**7 (128), + * to cover a time period of 1024 seconds into the future before wrapping. + */ +#define STTIMER_MINPOLL 3 /* log2 min poll interval (8 s) */ +#define STTIMER_SLOTTIME (1 << STTIMER_MINPOLL) +#define STTIMER_SLOTTIMEMASK (~(STTIMER_SLOTTIME - 1)) +#define STTIMER_NSLOTS (1 << 7) +#define STTIMER_SLOT(t) (&stt_data.stt_hash[(((t) >> STTIMER_MINPOLL) & \ + (STTIMER_NSLOTS - 1))]) + +struct st_timer_data { + spinlock_t stt_lock; + /* start time of the slot processed previously */ + cfs_time_t stt_prev_slot; + struct list_head stt_hash[STTIMER_NSLOTS]; + int stt_shuttingdown; +#ifdef __KERNEL__ + int stt_nthreads; +#endif +} stt_data; + +void +stt_add_timer (stt_timer_t *timer) +{ + struct list_head *pos; + + spin_lock(&stt_data.stt_lock); + +#ifdef __KERNEL__ + LASSERT (stt_data.stt_nthreads > 0); +#endif + LASSERT (!stt_data.stt_shuttingdown); + LASSERT (timer->stt_func != NULL); + LASSERT (list_empty(&timer->stt_list)); + LASSERT (cfs_time_after(timer->stt_expires, cfs_time_current_sec())); + + /* a simple insertion sort */ + list_for_each_prev (pos, STTIMER_SLOT(timer->stt_expires)) { + stt_timer_t *old = list_entry(pos, stt_timer_t, stt_list); + + if (cfs_time_aftereq(timer->stt_expires, old->stt_expires)) + break; + } + list_add(&timer->stt_list, pos); + + spin_unlock(&stt_data.stt_lock); +} + +/* + * The function returns whether it has deactivated a pending timer or not. + * (ie. del_timer() of an inactive timer returns 0, del_timer() of an + * active timer returns 1.) + * + * CAVEAT EMPTOR: + * When 0 is returned, it is possible that timer->stt_func _is_ running on + * another CPU. + */ +int +stt_del_timer (stt_timer_t *timer) +{ + int ret = 0; + + spin_lock(&stt_data.stt_lock); + +#ifdef __KERNEL__ + LASSERT (stt_data.stt_nthreads > 0); +#endif + LASSERT (!stt_data.stt_shuttingdown); + + if (!list_empty(&timer->stt_list)) { + ret = 1; + list_del_init(&timer->stt_list); + } + + spin_unlock(&stt_data.stt_lock); + return ret; +} + +/* called with stt_data.stt_lock held */ +int +stt_expire_list (struct list_head *slot, cfs_time_t now) +{ + int expired = 0; + stt_timer_t *timer; + + while (!list_empty(slot)) { + timer = list_entry(slot->next, stt_timer_t, stt_list); + + if (cfs_time_after(timer->stt_expires, now)) + break; + + list_del_init(&timer->stt_list); + spin_unlock(&stt_data.stt_lock); + + expired++; + (*timer->stt_func) (timer->stt_data); + + spin_lock(&stt_data.stt_lock); + } + + return expired; +} + +int +stt_check_timers (cfs_time_t *last) +{ + int expired = 0; + cfs_time_t now; + cfs_time_t this_slot; + + now = cfs_time_current_sec(); + this_slot = now & STTIMER_SLOTTIMEMASK; + + spin_lock(&stt_data.stt_lock); + + while (cfs_time_aftereq(this_slot, *last)) { + expired += stt_expire_list(STTIMER_SLOT(this_slot), now); + this_slot = cfs_time_sub(this_slot, STTIMER_SLOTTIME); + } + + *last = now & STTIMER_SLOTTIMEMASK; + spin_unlock(&stt_data.stt_lock); + return expired; +} + +#ifdef __KERNEL__ + +int +stt_timer_main (void *arg) +{ + UNUSED(arg); + + cfs_daemonize("st_timer"); + cfs_block_allsigs(); + + while (!stt_data.stt_shuttingdown) { + stt_check_timers(&stt_data.stt_prev_slot); + + set_current_state(CFS_TASK_INTERRUPTIBLE); + cfs_schedule_timeout(CFS_TASK_INTERRUPTIBLE, + cfs_time_seconds(STTIMER_SLOTTIME)); + } + + spin_lock(&stt_data.stt_lock); + stt_data.stt_nthreads--; + spin_unlock(&stt_data.stt_lock); + return 0; +} + +int +stt_start_timer_thread (void) +{ + long pid; + + LASSERT (!stt_data.stt_shuttingdown); + + pid = cfs_kernel_thread(stt_timer_main, NULL, 0); + if (pid < 0) + return (int)pid; + + spin_lock(&stt_data.stt_lock); + stt_data.stt_nthreads++; + spin_unlock(&stt_data.stt_lock); + return 0; +} + +#else /* !__KERNEL__ */ + +int +stt_check_events (void) +{ + return stt_check_timers(&stt_data.stt_prev_slot); +} + +int +stt_poll_interval (void) +{ + return STTIMER_SLOTTIME; +} + +#endif + +int +stt_startup (void) +{ + int rc = 0; + int i; + + stt_data.stt_shuttingdown = 0; + stt_data.stt_prev_slot = cfs_time_current_sec() & STTIMER_SLOTTIMEMASK; + + spin_lock_init(&stt_data.stt_lock); + for (i = 0; i < STTIMER_NSLOTS; i++) + CFS_INIT_LIST_HEAD(&stt_data.stt_hash[i]); + +#ifdef __KERNEL__ + stt_data.stt_nthreads = 0; + rc = stt_start_timer_thread(); + if (rc != 0) + CERROR ("Can't spawn timer, stt_startup() has failed: %d\n", rc); +#endif + + return rc; +} + +void +stt_shutdown (void) +{ + int i; + + spin_lock(&stt_data.stt_lock); + + for (i = 0; i < STTIMER_NSLOTS; i++) + LASSERT (list_empty(&stt_data.stt_hash[i])); + + stt_data.stt_shuttingdown = 1; + +#ifdef __KERNEL__ + lst_wait_until(stt_data.stt_nthreads == 0, stt_data.stt_lock, + "waiting for %d threads to terminate\n", + stt_data.stt_nthreads); +#endif + + spin_unlock(&stt_data.stt_lock); + return; +} diff --git a/lnet/selftest/timer.h b/lnet/selftest/timer.h new file mode 100644 index 0000000..c88027c --- /dev/null +++ b/lnet/selftest/timer.h @@ -0,0 +1,23 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2001, 2002 Cluster File Systems, Inc. + * Author: Isaac Huang + * + */ +#ifndef __SELFTEST_TIMER_H__ +#define __SELFTEST_TIMER_H__ + +typedef struct { + struct list_head stt_list; + cfs_time_t stt_expires; + void (*stt_func) (void *); + void *stt_data; +} stt_timer_t; + +void stt_add_timer (stt_timer_t *timer); +int stt_del_timer (stt_timer_t *timer); +int stt_startup (void); +void stt_shutdown (void); + +#endif /* __SELFTEST_TIMER_H__ */ diff --git a/lnet/selftest/workitem.c b/lnet/selftest/workitem.c new file mode 100644 index 0000000..0ed8350 --- /dev/null +++ b/lnet/selftest/workitem.c @@ -0,0 +1,343 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Copyright (C) 2001, 2002 Cluster File Systems, Inc. + * Author: Isaac Huang + * + */ +#define DEBUG_SUBSYSTEM S_LNET + +#include +#include +#include +#include "selftest.h" + + +struct smoketest_workitem { + struct list_head wi_runq; /* concurrent workitems */ + struct list_head wi_serial_runq; /* serialised workitems */ + cfs_waitq_t wi_waitq; /* where schedulers sleep */ + cfs_waitq_t wi_serial_waitq; /* where serial scheduler sleep */ + spinlock_t wi_lock; /* serialize */ + int wi_shuttingdown; + int wi_nthreads; +} swi_data; + +static inline int +swi_sched_cansleep (struct list_head *q) +{ + int rc; + + spin_lock(&swi_data.wi_lock); + + rc = !swi_data.wi_shuttingdown && list_empty(q); + + spin_unlock(&swi_data.wi_lock); + return rc; +} + +/* XXX: + * 0. it only works when called from wi->wi_action. + * 1. when it returns no one shall try to schedule the workitem. + */ +void +swi_kill_workitem (swi_workitem_t *wi) +{ + LASSERT (!in_interrupt()); /* because we use plain spinlock */ + LASSERT (!swi_data.wi_shuttingdown); + + spin_lock(&swi_data.wi_lock); + +#ifdef __KERNEL__ + LASSERT (wi->wi_running); +#endif + + if (wi->wi_scheduled) { /* cancel pending schedules */ + LASSERT (!list_empty(&wi->wi_list)); + list_del_init(&wi->wi_list); + } + + LASSERT (list_empty(&wi->wi_list)); + wi->wi_scheduled = 1; /* LBUG future schedule attempts */ + + spin_unlock(&swi_data.wi_lock); + return; +} + +void +swi_schedule_workitem (swi_workitem_t *wi) +{ + LASSERT (!in_interrupt()); /* because we use plain spinlock */ + LASSERT (!swi_data.wi_shuttingdown); + + spin_lock(&swi_data.wi_lock); + + if (!wi->wi_scheduled) { + LASSERT (list_empty(&wi->wi_list)); + + wi->wi_scheduled = 1; + list_add_tail(&wi->wi_list, &swi_data.wi_runq); + cfs_waitq_signal(&swi_data.wi_waitq); + } + + LASSERT (!list_empty(&wi->wi_list)); + spin_unlock(&swi_data.wi_lock); + return; +} + +/* + * Workitem scheduled by this function is strictly serialised not only with + * itself, but also with others scheduled this way. + * + * Now there's only one static serialised queue, but in the future more might + * be added, and even dynamic creation of serialised queues might be supported. + */ +void +swi_schedule_serial_workitem (swi_workitem_t *wi) +{ + LASSERT (!in_interrupt()); /* because we use plain spinlock */ + LASSERT (!swi_data.wi_shuttingdown); + + spin_lock(&swi_data.wi_lock); + + if (!wi->wi_scheduled) { + LASSERT (list_empty(&wi->wi_list)); + + wi->wi_scheduled = 1; + list_add_tail(&wi->wi_list, &swi_data.wi_serial_runq); + cfs_waitq_signal(&swi_data.wi_serial_waitq); + } + + LASSERT (!list_empty(&wi->wi_list)); + spin_unlock(&swi_data.wi_lock); + return; +} + +#ifdef __KERNEL__ + +int +swi_scheduler_main (void *arg) +{ + int id = (long) arg; + char name[16]; + + snprintf(name, sizeof(name), "swi_sd%03d", id); + cfs_daemonize(name); + cfs_block_allsigs(); + + spin_lock(&swi_data.wi_lock); + + while (!swi_data.wi_shuttingdown) { + int nloops = 0; + int rc; + swi_workitem_t *wi; + + while (!list_empty(&swi_data.wi_runq) && + nloops < SWI_RESCHED) { + wi = list_entry(swi_data.wi_runq.next, + swi_workitem_t, wi_list); + list_del_init(&wi->wi_list); + + LASSERT (wi->wi_scheduled); + + nloops++; + if (wi->wi_running) { + list_add_tail(&wi->wi_list, &swi_data.wi_runq); + continue; + } + + wi->wi_running = 1; + wi->wi_scheduled = 0; + spin_unlock(&swi_data.wi_lock); + + rc = (*wi->wi_action) (wi); + + spin_lock(&swi_data.wi_lock); + if (rc == 0) /* wi still active */ + wi->wi_running = 0; + } + + spin_unlock(&swi_data.wi_lock); + + if (nloops < SWI_RESCHED) + wait_event_interruptible_exclusive( + swi_data.wi_waitq, + !swi_sched_cansleep(&swi_data.wi_runq)); + else + our_cond_resched(); + + spin_lock(&swi_data.wi_lock); + } + + swi_data.wi_nthreads--; + spin_unlock(&swi_data.wi_lock); + return 0; +} + +int +swi_serial_scheduler_main (void *arg) +{ + UNUSED (arg); + + cfs_daemonize("swi_serial_sd"); + cfs_block_allsigs(); + + spin_lock(&swi_data.wi_lock); + + while (!swi_data.wi_shuttingdown) { + int nloops = 0; + int rc; + swi_workitem_t *wi; + + while (!list_empty(&swi_data.wi_serial_runq) && + nloops < SWI_RESCHED) { + wi = list_entry(swi_data.wi_serial_runq.next, + swi_workitem_t, wi_list); + list_del_init(&wi->wi_list); + + LASSERT (!wi->wi_running); + LASSERT (wi->wi_scheduled); + + nloops++; + wi->wi_running = 1; + wi->wi_scheduled = 0; + spin_unlock(&swi_data.wi_lock); + + rc = (*wi->wi_action) (wi); + + spin_lock(&swi_data.wi_lock); + if (rc == 0) /* wi still active */ + wi->wi_running = 0; + } + + spin_unlock(&swi_data.wi_lock); + + if (nloops < SWI_RESCHED) + wait_event_interruptible_exclusive( + swi_data.wi_serial_waitq, + !swi_sched_cansleep(&swi_data.wi_serial_runq)); + else + our_cond_resched(); + + spin_lock(&swi_data.wi_lock); + } + + swi_data.wi_nthreads--; + spin_unlock(&swi_data.wi_lock); + return 0; +} + +int +swi_start_thread (int (*func) (void*), void *arg) +{ + long pid; + + LASSERT (!swi_data.wi_shuttingdown); + + pid = cfs_kernel_thread(func, arg, 0); + if (pid < 0) + return (int)pid; + + spin_lock(&swi_data.wi_lock); + swi_data.wi_nthreads++; + spin_unlock(&swi_data.wi_lock); + return 0; +} + +#else /* __KERNEL__ */ + +int +swi_check_events (void) +{ + int n = 0; + swi_workitem_t *wi; + struct list_head *q; + + spin_lock(&swi_data.wi_lock); + + for (;;) { + if (!list_empty(&swi_data.wi_serial_runq)) + q = &swi_data.wi_serial_runq; + else if (!list_empty(&swi_data.wi_runq)) + q = &swi_data.wi_runq; + else + break; + + wi = list_entry(q->next, swi_workitem_t, wi_list); + list_del_init(&wi->wi_list); + + LASSERT (wi->wi_scheduled); + wi->wi_scheduled = 0; + spin_unlock(&swi_data.wi_lock); + + n++; + (*wi->wi_action) (wi); + + spin_lock(&swi_data.wi_lock); + } + + spin_unlock(&swi_data.wi_lock); + return n; +} + +#endif + +int +swi_startup (void) +{ + int i; + int rc; + + swi_data.wi_nthreads = 0; + swi_data.wi_shuttingdown = 0; + spin_lock_init(&swi_data.wi_lock); + cfs_waitq_init(&swi_data.wi_waitq); + cfs_waitq_init(&swi_data.wi_serial_waitq); + CFS_INIT_LIST_HEAD(&swi_data.wi_runq); + CFS_INIT_LIST_HEAD(&swi_data.wi_serial_runq); + +#ifdef __KERNEL__ + rc = swi_start_thread(swi_serial_scheduler_main, NULL); + if (rc != 0) { + LASSERT (swi_data.wi_nthreads == 0); + CERROR ("Can't spawn serial workitem scheduler: %d\n", rc); + return rc; + } + + for (i = 0; i < num_online_cpus(); i++) { + rc = swi_start_thread(swi_scheduler_main, (void *) (long) i); + if (rc != 0) { + CERROR ("Can't spawn workitem scheduler: %d\n", rc); + swi_shutdown(); + return rc; + } + } +#else + UNUSED(i); + UNUSED(rc); +#endif + + return 0; +} + +void +swi_shutdown (void) +{ + spin_lock(&swi_data.wi_lock); + + LASSERT (list_empty(&swi_data.wi_runq)); + LASSERT (list_empty(&swi_data.wi_serial_runq)); + + swi_data.wi_shuttingdown = 1; + +#ifdef __KERNEL__ + cfs_waitq_broadcast(&swi_data.wi_waitq); + cfs_waitq_broadcast(&swi_data.wi_serial_waitq); + lst_wait_until(swi_data.wi_nthreads == 0, swi_data.wi_lock, + "waiting for %d threads to terminate\n", + swi_data.wi_nthreads); +#endif + + spin_unlock(&swi_data.wi_lock); + return; +} diff --git a/lnet/ulnds/socklnd/dispatch.h b/lnet/ulnds/socklnd/dispatch.h index 300f33b..510525e 100644 --- a/lnet/ulnds/socklnd/dispatch.h +++ b/lnet/ulnds/socklnd/dispatch.h @@ -41,4 +41,4 @@ when now(void); /* * hacking for CFS internal MPI testing */ -#define ENABLE_SELECT_DISPATCH +#undef ENABLE_SELECT_DISPATCH diff --git a/lnet/utils/.cvsignore b/lnet/utils/.cvsignore index 13c2683..eae12d5 100644 --- a/lnet/utils/.cvsignore +++ b/lnet/utils/.cvsignore @@ -7,4 +7,6 @@ ptlctl routerstat wirecheck gmlndnid +lst +lstclient .*.cmd diff --git a/lnet/utils/Makefile.am b/lnet/utils/Makefile.am index 9cd3f25..1e04b80 100644 --- a/lnet/utils/Makefile.am +++ b/lnet/utils/Makefile.am @@ -19,10 +19,19 @@ sbin_PROGRAMS = debugctl lib_LIBRARIES = libptlctl.a +if LIBLUSTRE +noinst_LIBRARIES += liblst.a +liblst_a_SOURCES = +endif + libptlctl_a_SOURCES = portals.c nidstrings.c debug.c l_ioctl.c parser.c parser.h if UTILS -sbin_PROGRAMS += ptlctl routerstat wirecheck +sbin_PROGRAMS += ptlctl routerstat wirecheck lst lstclient +if LIBLUSTRE +sbin_PROGRAMS += lstclient +endif + if BUILD_GMLND sbin_PROGRAMS += gmlndnid endif @@ -45,5 +54,29 @@ debugctl_SOURCES = debugctl.c debugctl_LDADD = -L. -lptlctl $(LIBREADLINE) $(LIBEFENCE) debugctl_DEPENDENCIES = libptlctl.a +lst_SOURCES = lst.c +lst_LDADD = -L. -lptlctl $(LIBREADLINE) $(LIBEFENCE) +lst_DEPENDENCIES = libptlctl.a + +LND_LIBS = +if BUILD_USOCKLND +LND_LIBS += $(top_builddir)/lnet/ulnds/socklnd/libsocklnd.a +endif +if BUILD_UPTLLND +LND_LIBS += $(top_builddir)/lnet/ulnds/ptllnd/libptllnd.a +endif + +if LIBLUSTRE +LIB_SELFTEST = $(top_builddir)/lnet/libcfs/libcfs.a $(top_builddir)/lnet/lnet/liblnet.a $(top_builddir)/lnet/selftest/libselftest.a +liblst.a : $(LIB_SELFTEST) + sh $(srcdir)/genlib.sh "$(LIBS)" "$(LND_LIBS)" "$(PTHREAD_LIBS)" + +lstclient_SOURCES = lstclient.c +lstclient_LDADD = -L. -lptlctl -llst $(LIBREADLINE) $(LIBEFENCE) $(PTHREAD_LIBS) +lstclient_DEPENDENCIES = libptlctl.a liblst.a +endif + nidstrings.c: @top_srcdir@/lnet/libcfs/nidstrings.c ln -sf $< $@ + +EXTRA_DIST = genlib.sh diff --git a/lnet/utils/genlib.sh b/lnet/utils/genlib.sh new file mode 100755 index 0000000..66acf6a --- /dev/null +++ b/lnet/utils/genlib.sh @@ -0,0 +1,41 @@ +#!/bin/bash +#set -xv +set -e + +AR=/usr/bin/ar +LD=/usr/bin/ld +RANLIB=/usr/bin/ranlib + +CWD=`pwd` + +LIBS=$1 +LND_LIBS=$2 +PTHREAD_LIBS=$3 + +# do cleanup at first +rm -f liblst.so + +ALL_OBJS= + +build_obj_list() { + _objs=`$AR -t $1/$2` + for _lib in $_objs; do + ALL_OBJS=$ALL_OBJS"$1/$_lib "; + done; +} + +# lnet components libs +build_obj_list ../../lnet/libcfs libcfs.a +if $(echo "$LND_LIBS" | grep "socklnd" >/dev/null) ; then + build_obj_list ../../lnet/ulnds/socklnd libsocklnd.a +fi +if $(echo "$LND_LIBS" | grep "ptllnd" >/dev/null) ; then + build_obj_list ../../lnet/ulnds/ptllnd libptllnd.a +fi +build_obj_list ../../lnet/lnet liblnet.a +build_obj_list ../../lnet/selftest libselftest.a + +# create static lib lustre +rm -f $CWD/liblst.a +$AR -cru $CWD/liblst.a $ALL_OBJS +$RANLIB $CWD/liblst.a diff --git a/lnet/utils/lnetunload b/lnet/utils/lnetunload index d9fd908..8141178 100755 --- a/lnet/utils/lnetunload +++ b/lnet/utils/lnetunload @@ -2,12 +2,17 @@ lnds=$(echo k{sock,qsw,gm,{open,i,v,o2,c}ib,ra,ptl,mx}lnd) +do_rmmod() { + mod=$1 + if grep "^$mod" /proc/modules >/dev/null 2>&1; then + rmmod $mod + fi +} + +do_rmmod lnet_selftest + if lctl network down > /dev/null 2>&1; then - for mod in $lnds; do - if grep "^$mod" /proc/modules >/dev/null 2>&1; then - rmmod $mod - fi - done + for mod in $lnds; do do_rmmod $mod; done rmmod lnet rmmod libcfs diff --git a/lnet/utils/lst.c b/lnet/utils/lst.c new file mode 100644 index 0000000..07d4b07 --- /dev/null +++ b/lnet/utils/lst.c @@ -0,0 +1,2989 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Author: Liang Zhen + * + * This file is part of Lustre, http://www.lustre.org + */ + +#include +#include +#include +#include +#include +#include +#include +#include "parser.h" + +static command_t lst_cmdlist[]; +static lst_sid_t session_id; +static int session_key; +static lstcon_trans_stat_t trans_stat; + +typedef struct list_string { + struct list_string *lstr_next; + int lstr_sz; + char lstr_str[0]; +} lstr_t; + +#define offsetof(typ,memb) ((unsigned long)((char *)&(((typ *)0)->memb))) + +static int alloc_count = 0; +static int alloc_nob = 0; + +lstr_t * +alloc_lstr(int sz) +{ + lstr_t *lstr = malloc(offsetof(lstr_t, lstr_str[sz])); + + if (lstr == NULL) { + fprintf(stderr, "Can't allocate lstr\n"); + abort(); + } + + alloc_nob += sz; + alloc_count++; + + lstr->lstr_str[0] = 0; + lstr->lstr_sz = sz; + return lstr; +} + +void +free_lstr(lstr_t *lstr) +{ + alloc_count--; + alloc_nob -= lstr->lstr_sz; + free(lstr); +} + +void +free_lstrs(lstr_t **list) +{ + lstr_t *lstr; + + while ((lstr = *list) != NULL) { + *list = lstr->lstr_next; + free_lstr(lstr); + } +} + +new_lstrs(lstr_t **list, char *prefix, char *postfix, + int lo, int hi, int stride) +{ + int n1 = strlen(prefix); + int n2 = strlen(postfix); + int sz = n1 + 20 + n2 + 1; + + do { + lstr_t *n = alloc_lstr(sz); + + snprintf(n->lstr_str, sz - 1, "%s%u%s", + prefix, lo, postfix); + + n->lstr_next = *list; + *list = n; + + lo += stride; + } while (lo <= hi); +} + +int +expand_lstr(lstr_t **list, lstr_t *l) +{ + int nob = strlen(l->lstr_str); + char *b1; + char *b2; + char *expr; + char *sep; + int x; + int y; + int z; + int n; + + b1 = strchr(l->lstr_str, '['); + if (b1 == NULL) { + l->lstr_next = *list; + *list = l; + return 0; + } + + b2 = strchr(b1, ']'); + if (b2 == NULL || b2 == b1 + 1) + return -1; + + *b1++ = 0; + *b2++ = 0; + expr = b1; + do { + + sep = strchr(expr, ','); + if (sep != NULL) + *sep++ = 0; + + nob = strlen(expr); + n = nob; + if (sscanf(expr, "%u%n", &x, &n) >= 1 && n == nob) { + /* simple number */ + new_lstrs(list, l->lstr_str, b2, x, x, 1); + continue; + } + + n = nob; + if (sscanf(expr, "%u-%u%n", &x, &y, &n) >= 2 && n == nob && + x < y) { + /* simple range */ + new_lstrs(list, l->lstr_str, b2, x, y, 1); + continue; + } + + n = nob; + if (sscanf(expr, "%u-%u/%u%n", &x, &y, &z, &n) >= 3 && n == nob && + x < y) { + /* strided range */ + new_lstrs(list, l->lstr_str, b2, x, y, z); + continue; + } + + /* syntax error */ + return -1; + } while ((expr = sep) != NULL); + + free_lstr(l); + + return 1; +} + +int +expand_strs(char *str, lstr_t **head) +{ + lstr_t *list = NULL; + lstr_t *nlist; + lstr_t *l; + int rc; + int expanded; + + l = alloc_lstr(strlen(str) + 1); + memcpy(l->lstr_str, str, strlen(str) + 1); + l->lstr_next = NULL; + list = l; + + do { + expanded = 0; + nlist = NULL; + + while ((l = list) != NULL) { + list = l->lstr_next; + + rc = expand_lstr(&nlist, l); + if (rc < 0) { + fprintf(stderr, "Syntax error in \"%s\"\n", str); + free_lstr(l); + break; + } + + expanded |= rc > 0; + } + + /* re-order onto 'list' */ + while ((l = nlist) != NULL) { + nlist = l->lstr_next; + l->lstr_next = list; + list = l; + } + + } while (expanded && rc > 0); + + if (rc >= 0) { + *head = list; + return 0; + } + + while ((l = list) != NULL) { + list = l->lstr_next; + + free_lstr(l); + } + return rc; +} + +int +lst_parse_nids(char *str, int *countp, lnet_process_id_t **idspp) +{ + lstr_t *head = NULL; + lstr_t *l; + int c = 0; + int i; + int rc; + + rc = expand_strs(str, &head); + if (rc != 0) + goto out; + + l = head; + while (l != NULL) { + l = l->lstr_next; + c++; + } + + *idspp = malloc(c * sizeof(lnet_process_id_t)); + if (*idspp == NULL) { + fprintf(stderr, "Out of memory\n"); + rc = -1; + } + + *countp = c; +out: + i = 0; + while ((l = head) != NULL) { + head = l->lstr_next; + + if (rc == 0) { + (*idspp)[i].nid = libcfs_str2nid(l->lstr_str); + if ((*idspp)[i].nid == LNET_NID_ANY) { + fprintf(stderr, "Invalid nid: %s\n", + l->lstr_str); + rc = -1; + } + + (*idspp)[i].pid = LUSTRE_LNET_PID; + i++; + } + + free_lstr(l); + } + + if (rc == 0) + return 0; + + free(*idspp); + *idspp = NULL; + + return rc; +} + +char * +lst_node_state2str(int state) +{ + if (state == LST_NODE_ACTIVE) + return "Active"; + if (state == LST_NODE_BUSY) + return "Busy"; + if (state == LST_NODE_DOWN) + return "Down"; + + return "Unknown"; +} + +int +lst_node_str2state(char *str) +{ + if (strcasecmp(str, "active") == 0) + return LST_NODE_ACTIVE; + if (strcasecmp(str, "busy") == 0) + return LST_NODE_BUSY; + if (strcasecmp(str, "down") == 0) + return LST_NODE_DOWN; + if (strcasecmp(str, "unknown") == 0) + return LST_NODE_UNKNOWN; + if (strcasecmp(str, "invalid") == 0) + return (LST_NODE_UNKNOWN | LST_NODE_DOWN | LST_NODE_BUSY); + + return -1; +} + +char * +lst_test_type2name(int type) +{ + if (type == LST_TEST_PING) + return "ping"; + if (type == LST_TEST_BULK) + return "brw"; + + return "unknown"; +} + +int +lst_test_name2type(char *name) +{ + if (strcasecmp(name, "ping") == 0) + return LST_TEST_PING; + if (strcasecmp(name, "brw") == 0) + return LST_TEST_BULK; + + return -1; +} + +void +lst_print_usage(char *cmd) +{ + Parser_printhelp(cmd); +} + +void +lst_print_error(char *sub, const char *def_format, ...) +{ + va_list ap; + + /* local error returned from kernel */ + switch (errno) { + case ESRCH: + fprintf(stderr, "No session exists\n"); + return; + case ESHUTDOWN: + fprintf(stderr, "Session is shutting down\n"); + return; + case EACCES: + fprintf(stderr, "Unmatched session key or not root\n"); + return; + case ENOENT: + fprintf(stderr, "Can't find %s in current session\n", sub); + return; + case EINVAL: + fprintf(stderr, "Invalid parameters list in command line\n"); + return; + case EFAULT: + fprintf(stderr, "Bad parameter address\n"); + return; + case EEXIST: + fprintf(stderr, "%s already exists\n", sub); + return; + default: + va_start(ap, def_format); + vfprintf(stderr, def_format, ap); + va_end(ap); + + return; + } +} + +void +lst_free_rpcent(struct list_head *head) +{ + lstcon_rpc_ent_t *ent; + + while (!list_empty(head)) { + ent = list_entry(head->next, lstcon_rpc_ent_t, rpe_link); + + list_del(&ent->rpe_link); + free(ent); + } +} + +int +lst_reset_rpcent(struct list_head *head) +{ + lstcon_rpc_ent_t *ent; + + list_for_each_entry(ent, head, rpe_link) { + ent->rpe_sid = LST_INVALID_SID; + ent->rpe_peer.nid = LNET_NID_ANY; + ent->rpe_peer.pid = LNET_PID_ANY; + ent->rpe_rpc_errno = ent->rpe_fwk_errno = 0; + } +} + +int +lst_alloc_rpcent(struct list_head *head, int count, int offset) +{ + lstcon_rpc_ent_t *ent; + int i; + + for (i = 0; i < count; i++) { + ent = malloc(offsetof(lstcon_rpc_ent_t, rpe_payload[offset])); + if (ent == NULL) { + lst_free_rpcent(head); + return -1; + } + + memset(ent, 0, offsetof(lstcon_rpc_ent_t, rpe_payload[offset])); + + ent->rpe_sid = LST_INVALID_SID; + ent->rpe_peer.nid = LNET_NID_ANY; + ent->rpe_peer.pid = LNET_PID_ANY; + list_add(&ent->rpe_link, head); + } + + return 0; +} + +void +lst_print_transerr(struct list_head *head, char *optstr) +{ + lstcon_rpc_ent_t *ent; + + list_for_each_entry(ent, head, rpe_link) { + if (ent->rpe_rpc_errno == 0 && ent->rpe_fwk_errno == 0) + continue; + + if (ent->rpe_rpc_errno != 0) { + fprintf(stderr, "%s RPC failed on %s: %s\n", + optstr, libcfs_id2str(ent->rpe_peer), + strerror(ent->rpe_rpc_errno)); + continue; + } + + fprintf(stderr, "%s failed on %s: %s\n", + optstr, libcfs_id2str(ent->rpe_peer), + strerror(ent->rpe_fwk_errno)); + } +} + +int +lst_ioctl(unsigned int opc, void *buf, int len) +{ + struct libcfs_ioctl_data data; + int rc; + + LIBCFS_IOC_INIT (data); + data.ioc_u32[0] = opc; + data.ioc_plen1 = len; + data.ioc_pbuf1 = (char *)buf; + data.ioc_plen2 = sizeof(trans_stat); + data.ioc_pbuf2 = (char *)&trans_stat; + + memset(&trans_stat, 0, sizeof(trans_stat)); + + rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_LNETST, &data); + + /* local error, no valid RPC result */ + if (rc != 0) + return -1; + + /* RPC error */ + if (trans_stat.trs_rpc_errno != 0) + return -2; + + /* Framework error */ + if (trans_stat.trs_fwk_errno != 0) + return -3; + + return 0; +} + +int +lst_new_session_ioctl (char *name, int timeout, int force, lst_sid_t *sid) +{ + lstio_session_new_args_t args = { + .lstio_ses_key = session_key, + .lstio_ses_timeout = timeout, + .lstio_ses_force = force, + .lstio_ses_idp = sid, + .lstio_ses_namep = name, + .lstio_ses_nmlen = strlen(name), + }; + + return lst_ioctl (LSTIO_SESSION_NEW, &args, sizeof(args)); +} + +int +jt_lst_new_session(int argc, char **argv) +{ + char buf[LST_NAME_SIZE]; + char *name; + int optidx = 0; + int timeout = 300; + int force = 0; + int c; + int rc; + + static struct option session_opts[] = + { + {"timeout", required_argument, 0, 't' }, + {"force", no_argument, 0, 'f' }, + {0, 0, 0, 0 } + }; + + if (session_key == 0) { + fprintf(stderr, + "Can't find env LST_SESSION or value is not valid\n"); + return -1; + } + + while (1) { + + c = getopt_long(argc, argv, "ft:", + session_opts, &optidx); + + if (c == -1) + break; + + switch (c) { + case 'f': + force = 1; + break; + case 't': + timeout = atoi(optarg); + break; + default: + lst_print_usage(argv[0]); + return -1; + } + } + + if (timeout <= 0) { + fprintf(stderr, "Invalid timeout value\n"); + return -1; + } + + if (optind == argc - 1) { + name = argv[optind ++]; + if (strlen(name) >= LST_NAME_SIZE) { + fprintf(stderr, "Name size is limited to %d\n", + LST_NAME_SIZE - 1); + return -1; + } + + } else if (optind == argc) { + char user[LST_NAME_SIZE]; + char host[LST_NAME_SIZE]; + struct passwd *pw = getpwuid(getuid()); + + if (pw == NULL) + snprintf(user, sizeof(user), "%d", (int)getuid()); + else + snprintf(user, sizeof(user), "%s", pw->pw_name); + + rc = gethostname(host, sizeof(host)); + if (rc != 0) + snprintf(host, sizeof(host), "unknown_host"); + + snprintf(buf, LST_NAME_SIZE, "%s@%s", user, host); + name = buf; + + } else { + lst_print_usage(argv[0]); + return -1; + } + + rc = lst_new_session_ioctl(name, timeout, force, &session_id); + + if (rc != 0) { + lst_print_error("session", "Failed to create session: %s\n", + strerror(errno)); + return rc; + } + + fprintf(stdout, "SESSION: %s TIMEOUT: %d FORCE: %s\n", + name, timeout, force ? "Yes": "No"); + + return rc; +} + +int +lst_session_info_ioctl(char *name, int len, int *key, + lst_sid_t *sid, lstcon_ndlist_ent_t *ndinfo) +{ + lstio_session_info_args_t args = { + .lstio_ses_keyp = key, + .lstio_ses_idp = sid, + .lstio_ses_ndinfo = ndinfo, + .lstio_ses_nmlen = len, + .lstio_ses_namep = name, + }; + + return lst_ioctl(LSTIO_SESSION_INFO, &args, sizeof(args)); +} + +int +jt_lst_show_session(int argc, char **argv) +{ + lstcon_ndlist_ent_t ndinfo; + lst_sid_t sid; + char name[LST_NAME_SIZE]; + int key; + int rc; + + rc = lst_session_info_ioctl(name, LST_NAME_SIZE, &key, &sid, &ndinfo); + + if (rc != 0) { + lst_print_error("session", "Failed to show session: %s\n", + strerror(errno)); + return -1; + } + + fprintf(stdout, "%s ID: %Lu@%s, KEY: %d NODES: %d\n", + name, sid.ses_stamp, libcfs_nid2str(sid.ses_nid), + key, ndinfo.nle_nnode); + + return 0; +} + +int +lst_end_session_ioctl(void) +{ + lstio_session_end_args_t args = { + .lstio_ses_key = session_key, + }; + + return lst_ioctl (LSTIO_SESSION_END, &args, sizeof(args)); +} + +int +jt_lst_end_session(int argc, char **argv) +{ + int rc; + + if (session_key == 0) { + fprintf(stderr, + "Can't find env LST_SESSION or value is not valid\n"); + return -1; + } + + rc = lst_end_session_ioctl(); + + if (rc == 0) { + fprintf(stdout, "session is ended\n"); + return 0; + } + + if (rc == -1) { + lst_print_error("session", "Failed to end session: %s\n", + strerror(errno)); + return rc; + } + + if (trans_stat.trs_rpc_errno != 0) { + fprintf(stderr, + "[RPC] Failed to send %d session RPCs: %s\n", + lstcon_rpc_stat_failure(&trans_stat, 0), + strerror(trans_stat.trs_rpc_errno)); + } + + if (trans_stat.trs_fwk_errno != 0) { + fprintf(stderr, + "[FWK] Failed to end session on %d nodes: %s\n", + lstcon_sesop_stat_failure(&trans_stat, 0), + strerror(trans_stat.trs_fwk_errno)); + } + + return rc; +} + +int +lst_ping_ioctl(char *str, int type, int timeout, + int count, lnet_process_id_t *ids, struct list_head *head) +{ + lstio_debug_args_t args = { + .lstio_dbg_key = session_key, + .lstio_dbg_type = type, + .lstio_dbg_flags = 0, + .lstio_dbg_timeout = timeout, + .lstio_dbg_nmlen = (str == NULL) ? 0: strlen(str), + .lstio_dbg_namep = str, + .lstio_dbg_count = count, + .lstio_dbg_idsp = ids, + .lstio_dbg_resultp = head, + }; + + return lst_ioctl (LSTIO_DEBUG, &args, sizeof(args)); +} + +int +lst_get_node_count(int type, char *str, int *countp, lnet_process_id_t **idspp) +{ + char buf[LST_NAME_SIZE]; + lstcon_test_batch_ent_t ent; + lstcon_ndlist_ent_t *entp = &ent.tbe_cli_nle; + lst_sid_t sid; + int key; + int rc; + + switch (type) { + case LST_OPC_SESSION: + rc = lst_session_info_ioctl(buf, LST_NAME_SIZE, + &key, &sid, entp); + break; + + case LST_OPC_BATCHSRV: + entp = &ent.tbe_srv_nle; + case LST_OPC_BATCHCLI: + rc = lst_info_batch_ioctl(str, 0, 0, &ent, NULL, NULL, NULL); + break; + + case LST_OPC_GROUP: + rc = lst_info_group_ioctl(str, entp, NULL, NULL, NULL); + break; + + case LST_OPC_NODES: + rc = lst_parse_nids(str, &entp->nle_nnode, idspp) < 0 ? -1 : 0; + break; + + default: + rc = -1; + break; + } + + if (rc == 0) + *countp = entp->nle_nnode; + + return rc; +} + +int +jt_lst_ping(int argc, char **argv) +{ + struct list_head head; + lnet_process_id_t *ids = NULL; + lstcon_rpc_ent_t *ent = NULL; + char *str = NULL; + int optidx = 0; + int server = 0; + int timeout = 5; + int count = 0; + int type = 0; + int rc = 0; + int c; + int i; + + static struct option ping_opts[] = + { + {"session", no_argument, 0, 's' }, + {"server", no_argument, 0, 'v' }, + {"batch", required_argument, 0, 'b' }, + {"group", required_argument, 0, 'g' }, + {"nodes", required_argument, 0, 'n' }, + {"timeout", required_argument, 0, 't' }, + {0, 0, 0, 0 } + }; + + if (session_key == 0) { + fprintf(stderr, + "Can't find env LST_SESSION or value is not valid\n"); + return -1; + } + + while (1) { + + c = getopt_long(argc, argv, "g:b:n:t:sv", + ping_opts, &optidx); + + if (c == -1) + break; + + switch (c) { + case 's': + type = LST_OPC_SESSION; + break; + + case 'g': + type = LST_OPC_GROUP; + str = optarg; + break; + + case 'b': + type = LST_OPC_BATCHCLI; + str = optarg; + break; + + case 'n': + type = LST_OPC_NODES; + str = optarg; + break; + + case 't': + timeout = atoi(optarg); + break; + + case 'v': + server = 1; + break; + + default: + lst_print_usage(argv[0]); + return -1; + } + } + + if (type == 0 || timeout <= 0 || optind != argc) { + lst_print_usage(argv[0]); + return -1; + } + + if (type == LST_OPC_BATCHCLI && server) + type = LST_OPC_BATCHSRV; + + rc = lst_get_node_count(type, str, &count, &ids); + if (rc < 0) { + fprintf(stderr, "Failed to get count of nodes from %s: %s\n", + (str == NULL) ? "session" : str, strerror(errno)); + return -1; + } + + CFS_INIT_LIST_HEAD(&head); + + rc = lst_alloc_rpcent(&head, count, LST_NAME_SIZE); + if (rc != 0) { + fprintf(stderr, "Out of memory\n"); + goto out; + } + + if (count == 0) { + fprintf(stdout, "Target %s is empty\n", + (str == NULL) ? "session" : str); + goto out; + } + + rc = lst_ping_ioctl(str, type, timeout, count, ids, &head); + if (rc == -1) { /* local failure */ + lst_print_error("debug", "Failed to ping %s: %s\n", + (str == NULL) ? "session" : str, + strerror(errno)); + rc = -1; + goto out; + } + + /* ignore RPC errors and framwork errors */ + list_for_each_entry(ent, &head, rpe_link) { + fprintf(stdout, "\t%s: %s [session: %s id: %s]\n", + libcfs_id2str(ent->rpe_peer), + lst_node_state2str(ent->rpe_state), + (ent->rpe_state == LST_NODE_ACTIVE || + ent->rpe_state == LST_NODE_BUSY)? + (ent->rpe_rpc_errno == 0 ? + &ent->rpe_payload[0] : "Unknown") : + "", libcfs_nid2str(ent->rpe_sid.ses_nid)); + } + +out: + lst_free_rpcent(&head); + + if (ids != NULL) + free(ids); + + return rc; + +} + +int +lst_add_nodes_ioctl (char *name, int count, lnet_process_id_t *ids, + struct list_head *resultp) +{ + lstio_group_nodes_args_t args = { + .lstio_grp_key = session_key, + .lstio_grp_nmlen = strlen(name), + .lstio_grp_namep = name, + .lstio_grp_count = count, + .lstio_grp_idsp = ids, + .lstio_grp_resultp = resultp, + }; + + return lst_ioctl(LSTIO_NODES_ADD, &args, sizeof(args)); +} + +int +lst_add_group_ioctl (char *name) +{ + lstio_group_add_args_t args = { + .lstio_grp_key = session_key, + .lstio_grp_nmlen = strlen(name), + .lstio_grp_namep = name, + }; + + return lst_ioctl(LSTIO_GROUP_ADD, &args, sizeof(args)); +} + +int +jt_lst_add_group(int argc, char **argv) +{ + struct list_head head; + lnet_process_id_t *ids; + char *name; + int count; + int rc; + int i; + + if (session_key == 0) { + fprintf(stderr, + "Can't find env LST_SESSION or value is not valid\n"); + return -1; + } + + if (argc < 3) { + lst_print_usage(argv[0]); + return -1; + } + + name = argv[1]; + if (strlen(name) >= LST_NAME_SIZE) { + fprintf(stderr, "Name length is limited to %d\n", + LST_NAME_SIZE - 1); + return -1; + } + + rc = lst_add_group_ioctl(name); + if (rc != 0) { + lst_print_error("group", "Failed to add group %s: %s\n", + name, strerror(errno)); + return -1; + } + + CFS_INIT_LIST_HEAD(&head); + + for (i = 2; i < argc; i++) { + /* parse address list */ + rc = lst_parse_nids(argv[i], &count, &ids); + if (rc < 0) { + fprintf(stderr, "Ignore invalid id list %s\n", + argv[i]); + continue; + } + + if (count == 0) + continue; + + rc = lst_alloc_rpcent(&head, count, 0); + if (rc != 0) { + fprintf(stderr, "Out of memory\n"); + break; + } + + rc = lst_add_nodes_ioctl(name, count, ids, &head); + + free(ids); + + if (rc == 0) { + lst_free_rpcent(&head); + fprintf(stderr, "%s are added to session\n", argv[i]); + continue; + } + + if (rc == -1) { + lst_free_rpcent(&head); + lst_print_error("group", "Failed to add nodes %s: %s\n", + argv[i], strerror(errno)); + break; + } + + lst_print_transerr(&head, "create session"); + lst_free_rpcent(&head); + } + + return rc; +} + +int +lst_del_group_ioctl (char *name) +{ + lstio_group_del_args_t args = { + .lstio_grp_key = session_key, + .lstio_grp_nmlen = strlen(name), + .lstio_grp_namep = name, + }; + + return lst_ioctl(LSTIO_GROUP_DEL, &args, sizeof(args)); +} + +int +jt_lst_del_group(int argc, char **argv) +{ + int rc; + + if (session_key == 0) { + fprintf(stderr, + "Can't find env LST_SESSION or value is not valid\n"); + return -1; + } + + if (argc != 2) { + lst_print_usage(argv[0]); + return -1; + } + + rc = lst_del_group_ioctl(argv[1]); + if (rc == 0) { + fprintf(stdout, "Group is deleted\n"); + return 0; + } + + if (rc == -1) { + lst_print_error("group", "Failed to delete group: %s\n", + strerror(errno)); + return rc; + } + + fprintf(stderr, "Group is deleted with some errors\n"); + + if (trans_stat.trs_rpc_errno != 0) { + fprintf(stderr, "[RPC] Failed to send %d end session RPCs: %s\n", + lstcon_rpc_stat_failure(&trans_stat, 0), + strerror(trans_stat.trs_rpc_errno)); + } + + if (trans_stat.trs_fwk_errno != 0) { + fprintf(stderr, + "[FWK] Failed to end session on %d nodes: %s\n", + lstcon_sesop_stat_failure(&trans_stat, 0), + strerror(trans_stat.trs_fwk_errno)); + } + + return -1; +} + +int +lst_update_group_ioctl(int opc, char *name, int clean, int count, + lnet_process_id_t *ids, struct list_head *resultp) +{ + lstio_group_update_args_t args = { + .lstio_grp_key = session_key, + .lstio_grp_opc = opc, + .lstio_grp_args = clean, + .lstio_grp_nmlen = strlen(name), + .lstio_grp_namep = name, + .lstio_grp_count = count, + .lstio_grp_idsp = ids, + .lstio_grp_resultp = resultp, + }; + + return lst_ioctl(LSTIO_GROUP_UPDATE, &args, sizeof(args)); +} + +int +jt_lst_update_group(int argc, char **argv) +{ + struct list_head head; + lnet_process_id_t *ids = NULL; + char *str = NULL; + char *grp = NULL; + int optidx = 0; + int count = 0; + int clean = 0; + int opc = 0; + int rc; + int c; + + static struct option update_group_opts[] = + { + {"refresh", no_argument, 0, 'f' }, + {"clean", required_argument, 0, 'c' }, + {"remove", required_argument, 0, 'r' }, + {0, 0, 0, 0 } + }; + + if (session_key == 0) { + fprintf(stderr, + "Can't find env LST_SESSION or value is not valid\n"); + return -1; + } + + while (1) { + c = getopt_long(argc, argv, "fc:r:", + update_group_opts, &optidx); + + /* Detect the end of the options. */ + if (c == -1) + break; + + switch (c) { + case 'f': + if (opc != 0) { + lst_print_usage(argv[0]); + return -1; + } + opc = LST_GROUP_REFRESH; + break; + + case 'r': + if (opc != 0) { + lst_print_usage(argv[0]); + return -1; + } + opc = LST_GROUP_RMND; + str = optarg; + break; + + case 'c': + clean = lst_node_str2state(optarg); + if (opc != 0 || clean <= 0) { + lst_print_usage(argv[0]); + return -1; + } + opc = LST_GROUP_CLEAN; + break; + + default: + lst_print_usage(argv[0]); + return -1; + } + } + + /* no OPC or group is specified */ + if (opc == 0 || optind != argc - 1) { + lst_print_usage(argv[0]); + return -1; + } + + grp = argv[optind]; + + CFS_INIT_LIST_HEAD(&head); + + if (opc == LST_GROUP_RMND || opc == LST_GROUP_REFRESH) { + rc = lst_get_node_count(opc == LST_GROUP_RMND ? LST_OPC_NODES : + LST_OPC_GROUP, + opc == LST_GROUP_RMND ? str : grp, + &count, &ids); + + if (rc != 0) { + fprintf(stderr, "Can't get count of nodes from %s: %s\n", + opc == LST_GROUP_RMND ? str : grp, + strerror(errno)); + return -1; + } + + rc = lst_alloc_rpcent(&head, count, 0); + if (rc != 0) { + fprintf(stderr, "Out of memory\n"); + free(ids); + return -1; + } + + } + + rc = lst_update_group_ioctl(opc, grp, clean, count, ids, &head); + + if (ids != NULL) + free(ids); + + if (rc == 0) { + lst_free_rpcent(&head); + return 0; + } + + if (rc == -1) { + lst_free_rpcent(&head); + lst_print_error("group", "Failed to update group: %s\n", + strerror(errno)); + return rc; + } + + lst_print_transerr(&head, "Updating group"); + + lst_free_rpcent(&head); + + return rc; +} + +int +lst_list_group_ioctl(int len, char *name, int idx) +{ + lstio_group_list_args_t args = { + .lstio_grp_key = session_key, + .lstio_grp_idx = idx, + .lstio_grp_nmlen = len, + .lstio_grp_namep = name, + }; + + return lst_ioctl(LSTIO_GROUP_LIST, &args, sizeof(args)); +} + +int +lst_info_group_ioctl(char *name, lstcon_ndlist_ent_t *gent, + int *idx, int *count, lstcon_node_ent_t *dents) +{ + lstio_group_info_args_t args = { + .lstio_grp_key = session_key, + .lstio_grp_nmlen = strlen(name), + .lstio_grp_namep = name, + .lstio_grp_entp = gent, + .lstio_grp_idxp = idx, + .lstio_grp_ndentp = count, + .lstio_grp_dentsp = dents, + }; + + return lst_ioctl(LSTIO_GROUP_INFO, &args, sizeof(args)); +} + +int +lst_list_group_all(void) +{ + char name[LST_NAME_SIZE]; + int rc; + int i; + + /* no group is specified, list name of all groups */ + for (i = 0; ; i++) { + rc = lst_list_group_ioctl(LST_NAME_SIZE, name, i); + if (rc == 0) { + fprintf(stdout, "%d) %s\n", i + 1, name); + continue; + } + + if (errno == ENOENT) + break; + + lst_print_error("group", "Failed to list group: %s\n", + strerror(errno)); + return -1; + } + + fprintf(stdout, "Total %d groups\n", i); + + return 0; +} + +#define LST_NODES_TITLE "\tACTIVE\tBUSY\tDOWN\tUNKNOWN\tTOTAL\n" + +int +jt_lst_list_group(int argc, char **argv) +{ + lstcon_ndlist_ent_t gent; + lstcon_node_ent_t *dents; + int optidx = 0; + int verbose = 0; + int active = 0; + int busy = 0; + int down = 0; + int unknown = 0; + int all = 0; + int count; + int index; + int i; + int j; + int c; + int rc; + + static struct option list_group_opts[] = + { + {"active", no_argument, 0, 'a' }, + {"busy", no_argument, 0, 'b' }, + {"down", no_argument, 0, 'd' }, + {"unknown", no_argument, 0, 'u' }, + {"all", no_argument, 0, 'l' }, + {0, 0, 0, 0 } + }; + + if (session_key == 0) { + fprintf(stderr, + "Can't find env LST_SESSION or value is not valid\n"); + return -1; + } + + while (1) { + c = getopt_long(argc, argv, "abdul", + list_group_opts, &optidx); + + if (c == -1) + break; + + switch (c) { + case 'a': + verbose = active = 1; + all = 0; + break; + case 'b': + verbose = busy = 1; + all = 0; + break; + case 'd': + verbose = down = 1; + all = 0; + break; + case 'u': + verbose = unknown = 1; + all = 0; + break; + case 'l': + verbose = all = 1; + break; + default: + lst_print_usage(argv[0]); + return -1; + } + } + + if (optind == argc) { + /* no group is specified, list name of all groups */ + rc = lst_list_group_all(); + + return rc; + } + + if (!verbose) + fprintf(stdout, LST_NODES_TITLE); + + /* list nodes in specified groups */ + for (i = optind; i < argc; i++) { + rc = lst_info_group_ioctl(argv[i], &gent, NULL, NULL, NULL); + if (rc != 0) { + if (errno == ENOENT) { + rc = 0; + break; + } + + lst_print_error("group", "Failed to list group\n", + strerror(errno)); + break; + } + + if (!verbose) { + fprintf(stdout, "\t%d\t%d\t%d\t%d\t%d\t%s\n", + gent.nle_nactive, gent.nle_nbusy, + gent.nle_ndown, gent.nle_nunknown, + gent.nle_nnode, argv[i]); + continue; + } + + fprintf(stdout, "Group [ %s ]\n", argv[i]); + + if (gent.nle_nnode == 0) { + fprintf(stdout, "No nodes found [ %s ]\n", argv[i]); + continue; + } + + count = gent.nle_nnode; + + dents = malloc(count * sizeof(lstcon_node_ent_t)); + if (dents == NULL) { + fprintf(stderr, "Failed to malloc: %s\n", + strerror(errno)); + return -1; + } + + index = 0; + rc = lst_info_group_ioctl(argv[i], &gent, &index, &count, dents); + if (rc != 0) { + lst_print_error("group", "Failed to list group: %s\n", + strerror(errno)); + free(dents); + return -1; + } + + for (j = 0, c = 0; j < count; j++) { + if (all || + ((active && dents[j].nde_state == LST_NODE_ACTIVE) || + (busy && dents[j].nde_state == LST_NODE_BUSY) || + (down && dents[j].nde_state == LST_NODE_DOWN) || + (unknown && dents[j].nde_state == LST_NODE_UNKNOWN))) { + + fprintf(stdout, "\t%s: %s\n", + libcfs_id2str(dents[j].nde_id), + lst_node_state2str(dents[j].nde_state)); + c++; + } + } + + fprintf(stdout, "Total %d nodes [ %s ]\n", c, argv[i]); + + free(dents); + } + + return rc; +} + +int +lst_stat_ioctl (char *name, int count, lnet_process_id_t *idsp, + int timeout, struct list_head *resultp) +{ + lstio_stat_args_t args = { + .lstio_sta_key = session_key, + .lstio_sta_timeout = timeout, + .lstio_sta_nmlen = strlen(name), + .lstio_sta_namep = name, + .lstio_sta_count = count, + .lstio_sta_idsp = idsp, + .lstio_sta_resultp = resultp, + }; + + return lst_ioctl (LSTIO_STAT_QUERY, &args, sizeof(args)); +} + +typedef struct { + struct list_head srp_link; + int srp_count; + char *srp_name; + lnet_process_id_t *srp_ids; + struct list_head srp_result[2]; +} lst_stat_req_param_t; + +static void +lst_stat_req_param_free(lst_stat_req_param_t *srp) +{ + int i; + + for (i = 0; i < 2; i++) + lst_free_rpcent(&srp->srp_result[i]); + + if (srp->srp_ids != NULL) + free(srp->srp_ids); + + free(srp); +} + +static int +lst_stat_req_param_alloc(char *name, lst_stat_req_param_t **srpp) +{ + lst_stat_req_param_t *srp = NULL; + int rc; + int i; + + srp = malloc(sizeof(*srp)); + if (srp == NULL) + return -ENOMEM; + + memset(srp, 0, sizeof(*srp)); + CFS_INIT_LIST_HEAD(&srp->srp_result[0]); + CFS_INIT_LIST_HEAD(&srp->srp_result[1]); + + rc = lst_get_node_count(LST_OPC_GROUP, name, + &srp->srp_count, NULL); + if (rc != 0) { + rc = lst_get_node_count(LST_OPC_NODES, name, + &srp->srp_count, &srp->srp_ids); + } + + if (rc != 0) { + fprintf(stderr, + "Failed to get count of nodes from %s\n", name); + lst_stat_req_param_free(srp); + + return rc; + } + + srp->srp_name = name; + + for (i = 0; i < 2; i++) { + rc = lst_alloc_rpcent(&srp->srp_result[i], srp->srp_count, + sizeof(sfw_counters_t) + + sizeof(srpc_counters_t) + + sizeof(lnet_counters_t)); + if (rc != 0) { + fprintf(stderr, "Out of memory\n"); + break; + } + } + + if (rc == 0) { + *srpp = srp; + return 0; + } + + lst_stat_req_param_free(srp); + + return rc; +} + +typedef struct { + /* TODO */ +} lst_srpc_stat_result; + +#define LST_LNET_AVG 0 +#define LST_LNET_MIN 1 +#define LST_LNET_MAX 2 + +typedef struct { + float lnet_avg_sndrate; + float lnet_min_sndrate; + float lnet_max_sndrate; + float lnet_total_sndrate; + + float lnet_avg_rcvrate; + float lnet_min_rcvrate; + float lnet_max_rcvrate; + float lnet_total_rcvrate; + + float lnet_avg_sndperf; + float lnet_min_sndperf; + float lnet_max_sndperf; + float lnet_total_sndperf; + + float lnet_avg_rcvperf; + float lnet_min_rcvperf; + float lnet_max_rcvperf; + float lnet_total_rcvperf; + + int lnet_stat_count; +} lst_lnet_stat_result_t; + +lst_lnet_stat_result_t lnet_stat_result; + +static float +lst_lnet_stat_value(int bw, int send, int off) +{ + float *p; + + p = bw ? &lnet_stat_result.lnet_avg_sndperf : + &lnet_stat_result.lnet_avg_sndrate; + + if (!send) + p += 4; + + p += off; + + return *p; +} + +static void +lst_timeval_diff(struct timeval *tv1, + struct timeval *tv2, struct timeval *df) +{ + if (tv1->tv_usec >= tv2->tv_usec) { + df->tv_sec = tv1->tv_sec - tv2->tv_sec; + df->tv_usec = tv1->tv_usec - tv2->tv_usec; + return; + } + + df->tv_sec = tv1->tv_sec - 1 - tv2->tv_sec; + df->tv_usec = tv1->tv_sec + 1000000 - tv2->tv_usec; + + return; +} + +void +lst_cal_lnet_stat(float delta, lnet_counters_t *lnet_new, + lnet_counters_t *lnet_old) +{ + float perf; + float rate; + + perf = (float)(lnet_new->send_length - + lnet_old->send_length) / (1024 * 1024) / delta; + lnet_stat_result.lnet_total_sndperf += perf; + + if (lnet_stat_result.lnet_min_sndperf > perf || + lnet_stat_result.lnet_min_sndperf == 0) + lnet_stat_result.lnet_min_sndperf = perf; + + if (lnet_stat_result.lnet_max_sndperf < perf) + lnet_stat_result.lnet_max_sndperf = perf; + + perf = (float)(lnet_new->recv_length - + lnet_old->recv_length) / (1024 * 1024) / delta; + lnet_stat_result.lnet_total_rcvperf += perf; + + if (lnet_stat_result.lnet_min_rcvperf > perf || + lnet_stat_result.lnet_min_rcvperf == 0) + lnet_stat_result.lnet_min_rcvperf = perf; + + if (lnet_stat_result.lnet_max_rcvperf < perf) + lnet_stat_result.lnet_max_rcvperf = perf; + + rate = (lnet_new->send_count - lnet_old->send_count) / delta; + lnet_stat_result.lnet_total_sndrate += rate; + + if (lnet_stat_result.lnet_min_sndrate > rate || + lnet_stat_result.lnet_min_sndrate == 0) + lnet_stat_result.lnet_min_sndrate = rate; + + if (lnet_stat_result.lnet_max_sndrate < rate) + lnet_stat_result.lnet_max_sndrate = rate; + + rate = (lnet_new->recv_count - lnet_old->recv_count) / delta; + lnet_stat_result.lnet_total_rcvrate += rate; + + if (lnet_stat_result.lnet_min_rcvrate > rate || + lnet_stat_result.lnet_min_rcvrate == 0) + lnet_stat_result.lnet_min_rcvrate = rate; + + if (lnet_stat_result.lnet_max_rcvrate < rate) + lnet_stat_result.lnet_max_rcvrate = rate; + + lnet_stat_result.lnet_stat_count ++; + + lnet_stat_result.lnet_avg_sndrate = lnet_stat_result.lnet_total_sndrate / + lnet_stat_result.lnet_stat_count; + lnet_stat_result.lnet_avg_rcvrate = lnet_stat_result.lnet_total_rcvrate / + lnet_stat_result.lnet_stat_count; + + lnet_stat_result.lnet_avg_sndperf = lnet_stat_result.lnet_total_sndperf / + lnet_stat_result.lnet_stat_count; + lnet_stat_result.lnet_avg_rcvperf = lnet_stat_result.lnet_total_rcvperf / + lnet_stat_result.lnet_stat_count; + +} + +void +lst_print_lnet_stat(char *name, int bwrt, int rdwr, int type) +{ + int start1 = 0; + int end1 = 1; + int start2 = 0; + int end2 = 1; + int start3 = 0; + int end3 = 2; + int i; + int j; + + if (lnet_stat_result.lnet_stat_count == 0) + return; + + if (bwrt == 1) /* bw only */ + start1 = 1; + + if (bwrt == 2) /* rates only */ + end1 = 0; + + if (rdwr == 1) /* recv only */ + start2 = 1; + + if (rdwr == 2) /* send only */ + end2 = 0; + + for (i = start1; i <= end1; i++) { + fprintf(stdout, "[LNet %s of %s]\n", + i == 0 ? "Rates" : "Bandwidth", name); + + for (j = start2; j <= end2; j++) { + fprintf(stdout, "[%c] ", j == 0 ? 'W' : 'R'); + + if ((type & 1) != 0) { + fprintf(stdout, i == 0 ? "Avg: %-8.0f RPC/s " : + "Avg: %-8.2f MB/s ", + lst_lnet_stat_value(i, j, 0)); + } + + if ((type & 2) != 0) { + fprintf(stdout, i == 0 ? "Min: %-8.0f RPC/s " : + "Min: %-8.2f MB/s ", + lst_lnet_stat_value(i, j, 1)); + } + + if ((type & 4) != 0) { + fprintf(stdout, i == 0 ? "Max: %-8.0f RPC/s" : + "Max: %-8.2f MB/s", + lst_lnet_stat_value(i, j, 2)); + } + + fprintf(stdout, "\n"); + } + } +} + +void +lst_print_stat(char *name, struct list_head *resultp, + int idx, int lnet, int bwrt, int rdwr, int type) +{ + struct list_head tmp[2]; + lstcon_rpc_ent_t *new; + lstcon_rpc_ent_t *old; + sfw_counters_t *sfwk_new; + sfw_counters_t *sfwk_old; + srpc_counters_t *srpc_new; + srpc_counters_t *srpc_old; + lnet_counters_t *lnet_new; + lnet_counters_t *lnet_old; + struct timeval tv; + float delta; + int errcount = 0; + + CFS_INIT_LIST_HEAD(&tmp[0]); + CFS_INIT_LIST_HEAD(&tmp[1]); + + memset(&lnet_stat_result, 0, sizeof(lnet_stat_result)); + + while (!list_empty(&resultp[idx])) { + if (list_empty(&resultp[1 - idx])) { + fprintf(stderr, "Group is changed, re-run stat\n"); + break; + } + + new = list_entry(resultp[idx].next, lstcon_rpc_ent_t, rpe_link); + old = list_entry(resultp[1 - idx].next, lstcon_rpc_ent_t, rpe_link); + + /* first time get stats result, can't calculate diff */ + if (new->rpe_peer.nid == LNET_NID_ANY) + break; + + if (new->rpe_peer.nid != old->rpe_peer.nid || + new->rpe_peer.pid != old->rpe_peer.pid) { + /* Something wrong. i.e, somebody change the group */ + break; + } + + list_del(&new->rpe_link); + list_add_tail(&new->rpe_link, &tmp[idx]); + + list_del(&old->rpe_link); + list_add_tail(&old->rpe_link, &tmp[1 - idx]); + + if (new->rpe_rpc_errno != 0 || new->rpe_fwk_errno != 0 || + old->rpe_rpc_errno != 0 || old->rpe_fwk_errno != 0) { + errcount ++; + continue; + } + + sfwk_new = (sfw_counters_t *)&new->rpe_payload[0]; + sfwk_old = (sfw_counters_t *)&old->rpe_payload[0]; + + srpc_new = (srpc_counters_t *)((char *)sfwk_new + sizeof(*sfwk_new)); + srpc_old = (srpc_counters_t *)((char *)sfwk_old + sizeof(*sfwk_old)); + + lnet_new = (lnet_counters_t *)((char *)srpc_new + sizeof(*srpc_new)); + lnet_old = (lnet_counters_t *)((char *)srpc_old + sizeof(*srpc_old)); + + lst_timeval_diff(&new->rpe_stamp, &old->rpe_stamp, &tv); + + delta = tv.tv_sec + (float)tv.tv_usec/1000000; + + if (!lnet) /* TODO */ + continue; + + lst_cal_lnet_stat(delta, lnet_new, lnet_old); + } + + list_splice(&tmp[idx], &resultp[idx]); + list_splice(&tmp[1 - idx], &resultp[1 - idx]); + + if (errcount > 0) + fprintf(stdout, "Failed to stat on %d nodes\n", errcount); + + if (!lnet) /* TODO */ + return; + + lst_print_lnet_stat(name, bwrt, rdwr, type); +} + +int +jt_lst_stat(int argc, char **argv) +{ + struct list_head head; + lst_stat_req_param_t *srp; + lstcon_rpc_ent_t *ent; + char *name; + time_t last = 0; + int optidx = 0; + int count = 0; + int timeout = 5; /* default timeout, 5 sec */ + int delay = 5; /* default delay, 5 sec */ + int lnet = 1; /* lnet stat by default */ + int bwrt = 0; + int rdwr = 0; + int type = -1; + int idx = 0; + int rc; + int i; + int c; + + static struct option stat_opts[] = + { + {"timeout", required_argument, 0, 't' }, + {"delay" , required_argument, 0, 'd' }, + {"lnet" , no_argument, 0, 'l' }, + {"rpc" , no_argument, 0, 'c' }, + {"bw" , no_argument, 0, 'b' }, + {"rate" , no_argument, 0, 'a' }, + {"read" , no_argument, 0, 'r' }, + {"write" , no_argument, 0, 'w' }, + {"avg" , no_argument, 0, 'g' }, + {"min" , no_argument, 0, 'n' }, + {"max" , no_argument, 0, 'x' }, + {0, 0, 0, 0 } + }; + + if (session_key == 0) { + fprintf(stderr, + "Can't find env LST_SESSION or value is not valid\n"); + return -1; + } + + while (1) { + c = getopt_long(argc, argv, "t:d:lcbarwgnx", stat_opts, &optidx); + + if (c == -1) + break; + + switch (c) { + case 't': + timeout = atoi(optarg); + break; + case 'd': + delay = atoi(optarg); + break; + case 'l': + lnet = 1; + break; + case 'c': + lnet = 0; + break; + case 'b': + bwrt |= 1; + break; + case 'a': + bwrt |= 2; + break; + case 'r': + rdwr |= 1; + break; + case 'w': + rdwr |= 2; + break; + case 'g': + if (type == -1) { + type = 1; + break; + } + type |= 1; + break; + case 'n': + if (type == -1) { + type = 2; + break; + } + type |= 2; + break; + case 'x': + if (type == -1) { + type = 4; + break; + } + type |= 4; + break; + default: + lst_print_usage(argv[0]); + return -1; + } + } + + if (optind == argc) { + lst_print_usage(argv[0]); + return -1; + } + + if (timeout <= 0 || delay <= 0) { + fprintf(stderr, "Invalid timeout or delay value\n"); + return -1; + } + + CFS_INIT_LIST_HEAD(&head); + + while (optind < argc) { + name = argv[optind++]; + + rc = lst_stat_req_param_alloc(name, &srp); + if (rc != 0) + goto out; + + list_add_tail(&srp->srp_link, &head); + } + + while (1) { + time_t now = time(NULL); + + if (now - last < delay) { + sleep(delay - now + last); + time(&now); + } + + last = now; + + list_for_each_entry(srp, &head, srp_link) { + rc = lst_stat_ioctl(srp->srp_name, + srp->srp_count, srp->srp_ids, + timeout, &srp->srp_result[idx]); + if (rc == -1) { + lst_print_error("stat", "Failed to stat %s: %s\n", + name, strerror(errno)); + goto out; + } + + lst_print_stat(srp->srp_name, srp->srp_result, + idx, lnet, bwrt, rdwr, type); + + lst_reset_rpcent(&srp->srp_result[1 - idx]); + } + + idx = 1 - idx; + } + +out: + while (!list_empty(&head)) { + srp = list_entry(head.next, lst_stat_req_param_t, srp_link); + + list_del(&srp->srp_link); + lst_stat_req_param_free(srp); + } + + return rc; +} + +int +lst_add_batch_ioctl (char *name) +{ + lstio_batch_add_args_t args = { + .lstio_bat_key = session_key, + .lstio_bat_nmlen = strlen(name), + .lstio_bat_namep = name, + }; + + return lst_ioctl (LSTIO_BATCH_ADD, &args, sizeof(args)); +} + +int +jt_lst_add_batch(int argc, char **argv) +{ + char *name; + int rc; + + if (session_key == 0) { + fprintf(stderr, + "Can't find env LST_SESSION or value is not valid\n"); + return -1; + } + + if (argc != 2) { + lst_print_usage(argv[0]); + return -1; + } + + name = argv[1]; + if (strlen(name) >= LST_NAME_SIZE) { + fprintf(stderr, "Name length is limited to %d\n", + LST_NAME_SIZE - 1); + return -1; + } + + rc = lst_add_batch_ioctl(name); + if (rc == 0) + return 0; + + lst_print_error("batch", "Failed to create batch: %s\n", + strerror(errno)); + + return -1; +} + +int +lst_start_batch_ioctl (char *name, int timeout, struct list_head *resultp) +{ + lstio_batch_run_args_t args = { + .lstio_bat_key = session_key, + .lstio_bat_timeout = timeout, + .lstio_bat_nmlen = strlen(name), + .lstio_bat_namep = name, + .lstio_bat_resultp = resultp, + }; + + return lst_ioctl(LSTIO_BATCH_START, &args, sizeof(args)); +} + +int +jt_lst_start_batch(int argc, char **argv) +{ + struct list_head head; + char *batch; + int optidx = 0; + int timeout = 0; + int count = 0; + int rc; + int c; + + static struct option start_batch_opts[] = + { + {"timeout", required_argument, 0, 't' }, + {0, 0, 0, 0 } + }; + + if (session_key == 0) { + fprintf(stderr, + "Can't find env LST_SESSION or value is not valid\n"); + return -1; + } + + while (1) { + c = getopt_long(argc, argv, "t:", + start_batch_opts, &optidx); + + /* Detect the end of the options. */ + if (c == -1) + break; + + switch (c) { + case 't': + timeout = atoi(optarg); + break; + default: + lst_print_usage(argv[0]); + return -1; + } + } + + if (optind == argc) { + batch = LST_DEFAULT_BATCH; + + } else if (optind == argc - 1) { + batch = argv[optind]; + + } else { + lst_print_usage(argv[0]); + return -1; + } + + rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL); + if (rc != 0) { + fprintf(stderr, "Failed to get count of nodes from %s: %s\n", + batch, strerror(errno)); + return -1; + } + + CFS_INIT_LIST_HEAD(&head); + + rc = lst_alloc_rpcent(&head, count, 0); + if (rc != 0) { + fprintf(stderr, "Out of memory\n"); + return -1; + } + + rc = lst_start_batch_ioctl(batch, timeout, &head); + + if (rc == 0) { + fprintf(stdout, "%s is running now\n", batch); + lst_free_rpcent(&head); + return 0; + } + + if (rc == -1) { + lst_print_error("batch", "Failed to start batch: %s\n", + strerror(errno)); + lst_free_rpcent(&head); + return rc; + } + + lst_print_transerr(&head, "Run batch"); + + lst_free_rpcent(&head); + + return rc; +} + +int +lst_stop_batch_ioctl(char *name, int force, struct list_head *resultp) +{ + lstio_batch_stop_args_t args = { + .lstio_bat_key = session_key, + .lstio_bat_force = force, + .lstio_bat_nmlen = strlen(name), + .lstio_bat_namep = name, + .lstio_bat_resultp = resultp, + }; + + return lst_ioctl(LSTIO_BATCH_STOP, &args, sizeof(args)); +} + +int +jt_lst_stop_batch(int argc, char **argv) +{ + struct list_head head; + char *batch; + int force = 0; + int optidx; + int count; + int rc; + int c; + + static struct option stop_batch_opts[] = + { + {"force", no_argument, 0, 'f' }, + {0, 0, 0, 0 } + }; + + if (session_key == 0) { + fprintf(stderr, + "Can't find env LST_SESSION or value is not valid\n"); + return -1; + } + + while (1) { + c = getopt_long(argc, argv, "f", + stop_batch_opts, &optidx); + + /* Detect the end of the options. */ + if (c == -1) + break; + + switch (c) { + case 'f': + force = 1; + break; + default: + lst_print_usage(argv[0]); + return -1; + } + } + + if (optind == argc) { + batch = LST_DEFAULT_BATCH; + + } else if (optind == argc - 1) { + batch = argv[optind]; + + } else { + lst_print_usage(argv[0]); + return -1; + } + + rc = lst_get_node_count(LST_OPC_BATCHCLI, batch, &count, NULL); + if (rc != 0) { + fprintf(stderr, "Failed to get count of nodes from %s: %s\n", + batch, strerror(errno)); + return -1; + } + + CFS_INIT_LIST_HEAD(&head); + + rc = lst_alloc_rpcent(&head, count, 0); + if (rc != 0) { + fprintf(stderr, "Out of memory\n"); + return -1; + } + + rc = lst_stop_batch_ioctl(batch, force, &head); + if (rc != 0) + goto out; + + while (1) { + lst_reset_rpcent(&head); + + rc = lst_query_batch_ioctl(batch, 0, 0, 30, &head); + if (rc != 0) + goto out; + + if (lstcon_tsbqry_stat_run(&trans_stat, 0) == 0 && + lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) + break; + + fprintf(stdout, "%d batch in stopping\n", + lstcon_tsbqry_stat_run(&trans_stat, 0)); + sleep(1); + } + + fprintf(stdout, "Batch is stopped\n"); + lst_free_rpcent(&head); + + return 0; +out: + if (rc == -1) { + lst_print_error("batch", "Failed to stop batch: %s\n", + strerror(errno)); + lst_free_rpcent(&head); + return -1; + } + + lst_print_transerr(&head, "stop batch"); + + lst_free_rpcent(&head); + + return rc; +} + +int +lst_list_batch_ioctl(int len, char *name, int index) +{ + lstio_batch_list_args_t args = { + .lstio_bat_key = session_key, + .lstio_bat_idx = index, + .lstio_bat_nmlen = len, + .lstio_bat_namep = name, + }; + + return lst_ioctl(LSTIO_BATCH_LIST, &args, sizeof(args)); +} + +int +lst_info_batch_ioctl(char *batch, int test, int server, + lstcon_test_batch_ent_t *entp, int *idxp, + int *ndentp, lstcon_node_ent_t *dentsp) +{ + lstio_batch_info_args_t args = { + .lstio_bat_key = session_key, + .lstio_bat_nmlen = strlen(batch), + .lstio_bat_namep = batch, + .lstio_bat_server = server, + .lstio_bat_testidx = test, + .lstio_bat_entp = entp, + .lstio_bat_idxp = idxp, + .lstio_bat_ndentp = ndentp, + .lstio_bat_dentsp = dentsp, + }; + + return lst_ioctl(LSTIO_BATCH_INFO, &args, sizeof(args)); +} + +int +lst_list_batch_all(void) +{ + char name[LST_NAME_SIZE]; + int rc; + int i; + + for (i = 0; ; i++) { + rc = lst_list_batch_ioctl(LST_NAME_SIZE, name, i); + if (rc == 0) { + fprintf(stdout, "%d) %s\n", i + 1, name); + continue; + } + + if (errno == ENOENT) + break; + + lst_print_error("batch", "Failed to list batch: %s\n", + strerror(errno)); + return rc; + } + + fprintf(stdout, "Total %d batches\n", i); + + return 0; +} + +int +lst_list_tsb_nodes(char *batch, int test, int server, + int count, int active, int invalid) +{ + lstcon_node_ent_t *dents; + int index = 0; + int rc; + int c; + int i; + + if (count == 0) + return 0; + + /* verbose list, show nodes in batch or test */ + dents = malloc(count * sizeof(lstcon_node_ent_t)); + if (dents == NULL) { + fprintf(stdout, "Can't allocate memory\n"); + return -1; + } + + rc = lst_info_batch_ioctl(batch, test, server, + NULL, &index, &count, dents); + if (rc != 0) { + free(dents); + lst_print_error((test > 0) ? "test" : "batch", + (test > 0) ? "Failed to query test: %s\n" : + "Failed to query batch: %s\n", + strerror(errno)); + return -1; + } + + for (i = 0, c = 0; i < count; i++) { + if ((!active && dents[i].nde_state == LST_NODE_ACTIVE) || + (!invalid && (dents[i].nde_state == LST_NODE_BUSY || + dents[i].nde_state == LST_NODE_DOWN || + dents[i].nde_state == LST_NODE_UNKNOWN))) + continue; + + fprintf(stdout, "\t%s: %s\n", + libcfs_id2str(dents[i].nde_id), + lst_node_state2str(dents[i].nde_state)); + c++; + } + + fprintf(stdout, "Total %d nodes\n", c); + free(dents); + + return 0; +} + +int +jt_lst_list_batch(int argc, char **argv) +{ + lstcon_test_batch_ent_t ent; + char *batch = NULL; + int optidx = 0; + int verbose = 0; /* list nodes in batch or test */ + int invalid = 0; + int active = 0; + int server = 0; + int ntest = 0; + int test = 0; + int c = 0; + int i; + int rc; + + static struct option list_batch_opts[] = + { + {"test", required_argument, 0, 't' }, + {"invalid", no_argument, 0, 'i' }, + {"active", no_argument, 0, 'a' }, + {"all", no_argument, 0, 'l' }, + {"server", no_argument, 0, 's' }, + {0, 0, 0, 0 } + }; + + if (session_key == 0) { + fprintf(stderr, + "Can't find env LST_SESSION or value is not valid\n"); + return -1; + } + + while (1) { + c = getopt_long(argc, argv, "ailst:", + list_batch_opts, &optidx); + + if (c == -1) + break; + + switch (c) { + case 'a': + verbose = active = 1; + break; + case 'i': + verbose = invalid = 1; + break; + case 'l': + verbose = active = invalid = 1; + break; + case 's': + server = 1; + break; + case 't': + test = atoi(optarg); + ntest = 1; + break; + default: + lst_print_usage(argv[0]); + return -1; + } + } + + if (optind == argc) { + /* list all batches */ + rc = lst_list_batch_all(); + return rc; + } + + if (ntest == 1 && test <= 0) { + fprintf(stderr, "Invalid test id, test id starts from 1\n"); + return -1; + } + + if (optind != argc - 1) { + lst_print_usage(argv[0]); + return -1; + } + + batch = argv[optind]; + +loop: + /* show detail of specified batch or test */ + rc = lst_info_batch_ioctl(batch, test, server, + &ent, NULL, NULL, NULL); + if (rc != 0) { + lst_print_error((test > 0) ? "test" : "batch", + (test > 0) ? "Failed to query test: %s\n" : + "Failed to query batch: %s\n", + strerror(errno)); + return -1; + } + + if (verbose) { + /* list nodes in test or batch */ + rc = lst_list_tsb_nodes(batch, test, server, + server ? ent.tbe_srv_nle.nle_nnode : + ent.tbe_cli_nle.nle_nnode, + active, invalid); + return rc; + } + + /* only show number of hosts in batch or test */ + if (test == 0) { + fprintf(stdout, "Batch: %s Tests: %d State: %d\n", + batch, ent.u.tbe_batch.bae_ntest, + ent.u.tbe_batch.bae_state); + ntest = ent.u.tbe_batch.bae_ntest; + test = 1; /* starting from test 1 */ + + } else { + fprintf(stdout, + "\tTest %d(%s) (loop: %d, concurrency: %d)\n", + test, lst_test_type2name(ent.u.tbe_test.tse_type), + ent.u.tbe_test.tse_loop, + ent.u.tbe_test.tse_concur); + ntest --; + test ++; + } + + fprintf(stdout, LST_NODES_TITLE); + fprintf(stdout, "client\t%d\t%d\t%d\t%d\t%d\n" + "server\t%d\t%d\t%d\t%d\t%d\n", + ent.tbe_cli_nle.nle_nactive, + ent.tbe_cli_nle.nle_nbusy, + ent.tbe_cli_nle.nle_ndown, + ent.tbe_cli_nle.nle_nunknown, + ent.tbe_cli_nle.nle_nnode, + ent.tbe_srv_nle.nle_nactive, + ent.tbe_srv_nle.nle_nbusy, + ent.tbe_srv_nle.nle_ndown, + ent.tbe_srv_nle.nle_nunknown, + ent.tbe_srv_nle.nle_nnode); + + if (ntest != 0) + goto loop; + + return 0; +} + +int +lst_query_batch_ioctl(char *batch, int test, int server, + int timeout, struct list_head *head) +{ + lstio_batch_query_args_t args = { + .lstio_bat_key = session_key, + .lstio_bat_testidx = test, + .lstio_bat_client = !(server), + .lstio_bat_timeout = timeout, + .lstio_bat_nmlen = strlen(batch), + .lstio_bat_namep = batch, + .lstio_bat_resultp = head, + }; + + return lst_ioctl(LSTIO_BATCH_QUERY, &args, sizeof(args)); +} + +void +lst_print_tsb_verbose(struct list_head *head, + int active, int idle, int error) +{ + lstcon_rpc_ent_t *ent; + + list_for_each_entry(ent, head, rpe_link) { + if (ent->rpe_priv[0] == 0 && active) + continue; + + if (ent->rpe_priv[0] != 0 && idle) + continue; + + if (ent->rpe_fwk_errno == 0 && error) + continue; + + fprintf(stdout, "%s [%s]: %s\n", + libcfs_id2str(ent->rpe_peer), + lst_node_state2str(ent->rpe_state), + ent->rpe_rpc_errno != 0 ? + strerror(ent->rpe_rpc_errno) : + (ent->rpe_priv[0] > 0 ? "Running" : "Idle")); + } +} + +int +jt_lst_query_batch(int argc, char **argv) +{ + lstcon_test_batch_ent_t ent; + struct list_head head; + lstcon_rpc_ent_t *rent = NULL; + char *batch = NULL; + time_t last = 0; + int optidx = 0; + int index = 0; + int verbose = 0; + int server = 0; + int timeout = 5; /* default 5 seconds */ + int delay = 5; /* default 5 seconds */ + int loop = 1; /* default 1 loop */ + int active = 0; + int error = 0; + int idle = 0; + int count = 0; + int test = 0; + int rc = 0; + int c = 0; + int i; + + static struct option query_batch_opts[] = + { + {"timeout", required_argument, 0, 'o' }, + {"delay", required_argument, 0, 'd' }, + {"loop", required_argument, 0, 'c' }, + {"test", required_argument, 0, 't' }, + {"server", no_argument, 0, 's' }, + {"active", no_argument, 0, 'a' }, + {"idle", no_argument, 0, 'i' }, + {"error", no_argument, 0, 'e' }, + {"all", no_argument, 0, 'l' }, + {0, 0, 0, 0 } + }; + + if (session_key == 0) { + fprintf(stderr, + "Can't find env LST_SESSION or value is not valid\n"); + return -1; + } + + while (1) { + c = getopt_long(argc, argv, "o:d:c:t:saiel", + query_batch_opts, &optidx); + + /* Detect the end of the options. */ + if (c == -1) + break; + + switch (c) { + case 'o': + timeout = atoi(optarg); + break; + case 'd': + delay = atoi(optarg); + break; + case 'c': + loop = atoi(optarg); + break; + case 't': + test = atoi(optarg); + break; + case 's': + server = 1; + break; + case 'a': + active = verbose = 1; + break; + case 'i': + idle = verbose = 1; + break; + case 'e': + error = verbose = 1; + break; + case 'l': + verbose = 1; + break; + default: + lst_print_usage(argv[0]); + return -1; + } + } + + if (test < 0 || timeout <= 0 || delay <= 0 || loop <= 0) { + lst_print_usage(argv[0]); + return -1; + } + + if (optind == argc) { + batch = LST_DEFAULT_BATCH; + + } else if (optind == argc - 1) { + batch = argv[optind]; + + } else { + lst_print_usage(argv[0]); + return -1; + } + + + CFS_INIT_LIST_HEAD(&head); + + if (verbose) { + rc = lst_info_batch_ioctl(batch, test, server, + &ent, NULL, NULL, NULL); + if (rc != 0) { + fprintf(stderr, "Failed to query %s [%d]: %s\n", + batch, test, strerror(errno)); + return -1; + } + + count = server ? ent.tbe_srv_nle.nle_nnode : + ent.tbe_cli_nle.nle_nnode; + if (count == 0) { + fprintf(stdout, "Batch or test is empty\n"); + return 0; + } + } + + rc = lst_alloc_rpcent(&head, count, 0); + if (rc != 0) { + fprintf(stderr, "Out of memory\n"); + return rc; + } + + for (i = 0; i < loop; i++) { + time_t now = time(NULL); + + if (now - last < delay) { + sleep(delay - now + last); + time(&now); + } + + last = now; + + rc = lst_query_batch_ioctl(batch, test, + server, timeout, &head); + if (rc == -1) { + fprintf(stderr, "Failed to query batch: %s\n", + strerror(errno)); + break; + } + + if (verbose) { + /* Verbose mode */ + lst_print_tsb_verbose(&head, active, idle, error); + continue; + } + + fprintf(stdout, "%s [%d] ", batch, test); + + if (lstcon_rpc_stat_failure(&trans_stat, 0) != 0) { + fprintf(stdout, "%d of %d nodes are unknown, ", + lstcon_rpc_stat_failure(&trans_stat, 0), + lstcon_rpc_stat_total(&trans_stat, 0)); + } + + if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 && + lstcon_tsbqry_stat_run(&trans_stat, 0) == 0 && + lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) { + fprintf(stdout, "is stopped\n"); + continue; + } + + if (lstcon_rpc_stat_failure(&trans_stat, 0) == 0 && + lstcon_tsbqry_stat_idle(&trans_stat, 0) == 0 && + lstcon_tsbqry_stat_failure(&trans_stat, 0) == 0) { + fprintf(stdout, "is running\n"); + continue; + } + + fprintf(stdout, "stopped: %d , running: %d, failed: %d\n", + lstcon_tsbqry_stat_idle(&trans_stat, 0), + lstcon_tsbqry_stat_run(&trans_stat, 0), + lstcon_tsbqry_stat_failure(&trans_stat, 0)); + } + + lst_free_rpcent(&head); + + return rc; +} + +int +lst_parse_distribute(char *dstr, int *dist, int *span) +{ + *dist = atoi(dstr); + if (*dist <= 0) + return -1; + + dstr = strchr(dstr, ':'); + if (dstr == NULL) + return -1; + + *span = atoi(dstr + 1); + if (*span <= 0) + return -1; + + return 0; +} + +int +lst_get_test_param(char *test, int argc, char **argv, void **param, int *plen) +{ + lst_test_bulk_param_t *bulk = NULL; + lst_test_ping_param_t *ping = NULL; + int type; + int i = 0; + + type = lst_test_name2type(test); + if (type < 0) + return -EINVAL; + + switch (type) { + case LST_TEST_PING: + break; + + case LST_TEST_BULK: + if (i == argc) + return -EINVAL; + + bulk = malloc(sizeof(*bulk)); + if (bulk == NULL) + return -ENOMEM; + + memset(bulk, 0, sizeof(*bulk)); + + if (strcmp(argv[i], "w") == 0) + bulk->blk_opc = LST_BRW_WRITE; + else /* read by default */ + bulk->blk_opc = LST_BRW_READ; + + if (++i == argc) { + /* 1 page by default */ + bulk->blk_flags = LST_BRW_CHECK_NONE; + bulk->blk_npg = 1; + *param = bulk; + *plen = sizeof(*bulk); + break; + + } + + bulk->blk_npg = atoi(argv[i]); + if (bulk->blk_npg <= 0 || + bulk->blk_npg >= LNET_MAX_IOV) { + free(bulk); + return -EINVAL; + } + + if (++i == argc) { + bulk->blk_flags = LST_BRW_CHECK_NONE; + *param = bulk; + *plen = sizeof(*bulk); + break; + } + + /* posion pages */ + if (strcmp(argv[i], "s") == 0) + bulk->blk_flags = LST_BRW_CHECK_SIMPLE; + else if (strcmp(argv[i], "f") == 0) + bulk->blk_flags = LST_BRW_CHECK_FULL; + else + bulk->blk_flags = LST_BRW_CHECK_NONE; + + *param = bulk; + *plen = sizeof(*bulk); + + break; + + default: + break; + } + + /* TODO: parse more parameter */ + return type; +} + +int +lst_add_test_ioctl(char *batch, int type, int loop, int concur, + int dist, int span, char *sgrp, char *dgrp, + void *param, int plen, struct list_head *resultp) +{ + lstio_test_args_t args = { + .lstio_tes_key = session_key, + .lstio_tes_bat_nmlen = strlen(batch), + .lstio_tes_bat_name = batch, + .lstio_tes_type = type, + .lstio_tes_loop = loop, + .lstio_tes_concur = concur, + .lstio_tes_dist = dist, + .lstio_tes_span = span, + .lstio_tes_sgrp_nmlen = strlen(sgrp), + .lstio_tes_sgrp_name = sgrp, + .lstio_tes_dgrp_nmlen = strlen(dgrp), + .lstio_tes_dgrp_name = dgrp, + .lstio_tes_param_len = plen, + .lstio_tes_param = param, + .lstio_tes_resultp = resultp, + }; + + return lst_ioctl(LSTIO_TEST_ADD, &args, sizeof(args)); +} + +int +jt_lst_add_test(int argc, char **argv) +{ + struct list_head head; + char *batch = NULL; + char *test = NULL; + char *dstr = NULL; + char *from = NULL; + char *to = NULL; + void *param = NULL; + int optidx = 0; + int concur = 1; + int loop = -1; + int dist = 1; + int span = 1; + int plen = 0; + int fcount = 0; + int tcount = 0; + int type; + int rc; + int c; + + static struct option add_test_opts[] = + { + {"batch", required_argument, 0, 'b' }, + {"concurrency", required_argument, 0, 'c' }, + {"distribute", required_argument, 0, 'd' }, + {"from", required_argument, 0, 'f' }, + {"to", required_argument, 0, 't' }, + {"loop", required_argument, 0, 'l' }, + {0, 0, 0, 0 } + }; + + if (session_key == 0) { + fprintf(stderr, + "Can't find env LST_SESSION or value is not valid\n"); + return -1; + } + + while (1) { + c = getopt_long(argc, argv, "b:c:d:f:l:t:", + add_test_opts, &optidx); + + /* Detect the end of the options. */ + if (c == -1) + break; + + switch (c) { + case 'b': + batch = optarg; + break; + case 'c': + concur = atoi(optarg); + break; + case 'd': + dstr = optarg; + break; + case 'f': + from = optarg; + break; + case 'l': + loop = atoi(optarg); + break; + case 't': + to = optarg; + break; + default: + lst_print_usage(argv[0]); + return -1; + } + } + + if (optind == argc || from == NULL || to == NULL) { + lst_print_usage(argv[0]); + return -1; + } + + if (concur <= 0 || concur > LST_MAX_CONCUR) { + fprintf(stderr, "Invalid concurrency of test: %d\n", concur); + return -1; + } + + if (batch == NULL) + batch = LST_DEFAULT_BATCH; + + if (dstr != NULL) { + rc = lst_parse_distribute(dstr, &dist, &span); + if (rc != 0) { + fprintf(stderr, "Invalid distribution: %s\n", dstr); + return -1; + } + } + + test = argv[optind++]; + + argc -= optind; + argv += optind; + + type = lst_get_test_param(test, argc, argv, ¶m, &plen); + if (type < 0) { + fprintf(stderr, "Can't parse test (%s) parameter: %s\n", + test, strerror(-type)); + return -1; + } + + CFS_INIT_LIST_HEAD(&head); + + rc = lst_get_node_count(LST_OPC_GROUP, from, &fcount, NULL); + if (rc != 0) { + fprintf(stderr, "Can't get count of nodes from %s: %s\n", + from, strerror(errno)); + goto out; + } + + rc = lst_get_node_count(LST_OPC_GROUP, to, &tcount, NULL); + if (rc != 0) { + fprintf(stderr, "Can't get count of nodes from %s: %s\n", + to, strerror(errno)); + goto out; + } + + rc = lst_alloc_rpcent(&head, fcount > tcount ? fcount : tcount, 0); + if (rc != 0) { + fprintf(stderr, "Out of memory\n"); + goto out; + } + + rc = lst_add_test_ioctl(batch, type, loop, concur, + dist, span, from, to, param, plen, &head); + + if (rc == 0) { + fprintf(stdout, "Test was added successfully\n"); + goto out; + } + + if (rc == -1) { + lst_print_error("test", "Failed to add test: %s\n", + strerror(errno)); + goto out; + } + + lst_print_transerr(&head, "add test"); +out: + lst_free_rpcent(&head); + + if (param != NULL) + free(param); + + return rc; +} + +command_t lst_cmdlist[] = { + {"new_session", jt_lst_new_session, NULL, + "Usage: lst new_session [--timeout TIME] [--force] [NAME]" }, + {"end_session", jt_lst_end_session, NULL, + "Usage: lst end_session" }, + {"show_session", jt_lst_show_session, NULL, + "Usage: lst show_session" }, + {"ping", jt_lst_ping , NULL, + "Usage: lst ping [--group NAME] [--batch NAME] [--session] [--nodes IDS]" }, + {"add_group", jt_lst_add_group, NULL, + "Usage: lst group NAME IDs [IDs]..." }, + {"del_group", jt_lst_del_group, NULL, + "Usage: lst del_group NAME" }, + {"update_group", jt_lst_update_group, NULL, + "Usage: lst update_group NAME [--clean] [--refresh] [--remove IDs]" }, + {"list_group", jt_lst_list_group, NULL, + "Usage: lst list_group [--active] [--busy] [--down] [--unknown] GROUP ..." }, + {"stat", jt_lst_stat, NULL, + "Usage: lst stat [--bw] [--rate] [--read] [--write] [--max] [--min] [--avg] " + " [--timeout #] [--delay #] GROUP [GROUP]" }, + {"add_batch", jt_lst_add_batch, NULL, + "Usage: lst add_batch NAME" }, + {"run", jt_lst_start_batch, NULL, + "Usage: lst run [--timeout TIME] [NAME]" }, + {"stop", jt_lst_stop_batch, NULL, + "Usage: lst stop [--force] BATCH_NAME" }, + {"list_batch", jt_lst_list_batch, NULL, + "Usage: lst list_batch NAME [--test ID] [--server]" }, + {"query", jt_lst_query_batch, NULL, + "Usage: lst query [--test ID] [--server] [--timeout TIME] NAME" }, + {"add_test", jt_lst_add_test, NULL, + "Usage: lst add_test [--batch BATCH] [--loop #] [--concurrency #] " + " [--distribute #:#] [--from GROUP] [--to GROUP] TEST..." }, + {"help", Parser_help, 0, "help" }, + {0, 0, 0, NULL } +}; + +int +lst_initialize(void) +{ + char *key; + + key = getenv("LST_SESSION"); + + if (key == NULL) { + session_key = 0; + return 0; + } + + session_key = atoi(key); + + return 0; +} + +int +main(int argc, char **argv) +{ + int rc; + + setlinebuf(stdout); + + if (lst_initialize() < 0) + exit(0); + + if (ptl_initialize(argc, argv) < 0) + exit(0); + + Parser_init("lst > ", lst_cmdlist); + + if (argc != 1) + return Parser_execarg(argc - 1, argv + 1, lst_cmdlist); + + Parser_commands(); + + return 0; +} diff --git a/lnet/utils/lstclient.c b/lnet/utils/lstclient.c new file mode 100644 index 0000000..075872b --- /dev/null +++ b/lnet/utils/lstclient.c @@ -0,0 +1,205 @@ +/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- + * vim:expandtab:shiftwidth=8:tabstop=8: + * + * Author: Liang Zhen + * + * This file is part of Lustre, http://www.lustre.org + */ +#include +#include +#include +#include +#include +#include +#include +#include "../selftest/rpc.h" +#include "../selftest/selftest.h" + +static int lstjn_stopping = 0; +static int lstjn_intialized = 0; + +unsigned int libcfs_subsystem_debug = ~0 - (S_LNET | S_LND); +unsigned int libcfs_debug = 0; + +extern int sfw_session_removed(void); + +static struct option lstjn_options[] = +{ + {"sesid", required_argument, 0, 's' }, + {"group", required_argument, 0, 'g' }, + {0, 0, 0, 0 } +}; + +void +lstjn_stop (int sig) +{ + lstjn_stopping = 1; +} + +void +lstjn_rpc_done(srpc_client_rpc_t *rpc) +{ + if (!lstjn_intialized) + lstjn_intialized = 1; +} + +int +lstjn_join_session(char *ses, char *grp) +{ + lnet_process_id_t sesid; + srpc_client_rpc_t *rpc; + srpc_join_reqst_t *req; + srpc_join_reply_t *rep; + srpc_mksn_reqst_t *sreq; + srpc_mksn_reply_t *srep; + int rc; + + sesid.nid = libcfs_str2nid(ses); + sesid.pid = LUSTRE_LNET_PID; + + rpc = sfw_create_rpc(sesid, SRPC_SERVICE_JOIN, 0, + 0, lstjn_rpc_done, NULL); + if (rpc == NULL) { + fprintf(stderr, "Out of memory\n"); + return -1; + } + + req = &rpc->crpc_reqstmsg.msg_body.join_reqst; + + req->join_sid = LST_INVALID_SID; + strncpy(req->join_group, grp, LST_NAME_SIZE); + + sfw_post_rpc(rpc); + + for (;;) { + rc = selftest_wait_events(); + + if (lstjn_intialized) + break; + } + + if (rpc->crpc_status != 0) { + fprintf(stderr, "Failed to send RPC to console: %s\n", + strerror(rpc->crpc_status)); + srpc_client_rpc_decref(rpc); + return -1; + } + + sfw_unpack_message(&rpc->crpc_replymsg); + + rep = &rpc->crpc_replymsg.msg_body.join_reply; + if (rep->join_status != 0) { + fprintf(stderr, "Can't join session %s group %s: %s\n", + ses, grp, strerror(rep->join_status)); + srpc_client_rpc_decref(rpc); + return -1; + } + + sreq = &rpc->crpc_reqstmsg.msg_body.mksn_reqst; + sreq->mksn_sid = rep->join_sid; + sreq->mksn_force = 0; + strcpy(sreq->mksn_name, rep->join_session); + + srep = &rpc->crpc_replymsg.msg_body.mksn_reply; + + rc = sfw_make_session(sreq, srep); + if (rc != 0 || srep->mksn_status != 0) { + fprintf(stderr, "Can't create session: %d, %s\n", + rc, strerror(srep->mksn_status)); + srpc_client_rpc_decref(rpc); + return -1; + } + + fprintf(stdout, "Session %s, ID: %s, %Lu\n", + ses, libcfs_nid2str(rep->join_sid.ses_nid), + rep->join_sid.ses_stamp); + + srpc_client_rpc_decref(rpc); + + return 0; +} + +int +main(int argc, char **argv) +{ + char *ses = NULL; + char *grp = NULL; + int optidx; + int c; + int rc; + + while (1) { + c = getopt_long(argc, argv, "s:g:", + lstjn_options, &optidx); + + if (c == -1) + break; + + switch (c) { + case 's': + ses = optarg; + break; + case 'g': + grp = optarg; + break; + default: + fprintf(stderr, + "Usage: lstclient --sesid ID --group GROUP\n"); + return -1; + } + } + + if (optind != argc || grp == NULL || ses == NULL) { + fprintf(stderr, "Usage: lstclient --sesid ID --group GROUP\n"); + return -1; + } + + rc = libcfs_debug_init(5 * 1024 * 1024); + if (rc != 0) { + CERROR("libcfs_debug_init() failed: %d\n", rc); + return -1; + } + + rc = LNetInit(); + if (rc != 0) { + CERROR("LNetInit() failed: %d\n", rc); + libcfs_debug_cleanup(); + return -1; + } + + rc = lnet_selftest_init(); + if (rc != 0) { + fprintf(stderr, "Can't startup selftest\n"); + LNetFini(); + libcfs_debug_cleanup(); + + return -1; + } + + rc = lstjn_join_session(ses, grp); + if (rc != 0) + goto out; + + signal(SIGINT, lstjn_stop); + + fprintf(stdout, "Start handling selftest requests, Ctl-C to stop\n"); + + while (!lstjn_stopping) { + selftest_wait_events(); + + if (!sfw_session_removed()) + continue; + + fprintf(stdout, "Session ended\n"); + break; + } + +out: + lnet_selftest_fini(); + + LNetFini(); + + libcfs_debug_cleanup(); + + return rc; +} diff --git a/lnet/utils/parser.c b/lnet/utils/parser.c index 2f740c1..ee9604e 100644 --- a/lnet/utils/parser.c +++ b/lnet/utils/parser.c @@ -333,6 +333,10 @@ int Parser_commands(void) if (*s) { add_history(s); rc = execute_line(s); + + /* reset optind to 0 to tell getopt + * to reinitialize itself */ + optind = 0; } free(line);