From fe651f6b21a442436b1b3f3a2b14fa2eb119aa04 Mon Sep 17 00:00:00 2001 From: Liang Zhen Date: Mon, 5 Sep 2011 23:12:32 +0800 Subject: [PATCH] LU-521 lnet: make LST support variable page size LNet selftest can't support variable page size because it sends number of pages over the wire. We have to change wire format to fix this but it will have compatibility issue, so this patch also implemented "session features" for LST to resolve compatibility issue, only new version LST can understand new bulk RPC format, although new LST nodes still can communicate with old LST nodes. The Variable page size feature can be turned on by setting LST_FEATURES to LST_FEAT_BULK_LEN (1). This feature is off by default. Please see the Jira ticket (LU-521) for more details. Change-Id: I4a552a3310cf0ed0a2f5ae29eaf789469e1c245a Signed-off-by: Liang Zhen Signed-off-by: Doug Oucharek Reviewed-on: http://review.whamcloud.com/1338 Reviewed-by: Andreas Dilger Tested-by: Hudson Tested-by: Maloo --- lnet/include/lnet/lnetst.h | 12 ++ lnet/selftest/brw_test.c | 172 +++++++++++++++++-------- lnet/selftest/conctl.c | 42 ++++-- lnet/selftest/conrpc.c | 309 ++++++++++++++++++++++++++++----------------- lnet/selftest/conrpc.h | 25 ++-- lnet/selftest/console.c | 115 ++++++++++++----- lnet/selftest/console.h | 23 +++- lnet/selftest/framework.c | 161 +++++++++++++++++------ lnet/selftest/ping_test.c | 44 +++++-- lnet/selftest/rpc.c | 132 ++++++++++--------- lnet/selftest/rpc.h | 78 ++++++++---- lnet/selftest/selftest.h | 26 ++-- lnet/utils/lst.c | 149 ++++++++++++++-------- lnet/utils/lstclient.c | 99 +++++++++------ 14 files changed, 922 insertions(+), 465 deletions(-) diff --git a/lnet/include/lnet/lnetst.h b/lnet/include/lnet/lnetst.h index 61aa65e..6240fdf 100644 --- a/lnet/include/lnet/lnetst.h +++ b/lnet/include/lnet/lnetst.h @@ -43,6 +43,12 @@ #include #include +#define LST_FEAT_NONE (0) +#define LST_FEAT_BULK_LEN (1 << 0) /* enable variable page size */ + +#define LST_FEATS_EMPTY (LST_FEAT_NONE) +#define LST_FEATS_MASK (LST_FEAT_NONE | LST_FEAT_BULK_LEN) + #define LST_NAME_SIZE 32 /* max name buffer length */ #define LSTIO_DEBUG 0xC00 /* debug */ @@ -233,6 +239,8 @@ typedef struct { int lstio_ses_key; /* IN: local key */ int lstio_ses_timeout; /* IN: session timeout */ int lstio_ses_force; /* IN: force create ? */ + /** IN: session features */ + unsigned lstio_ses_feats; lst_sid_t *lstio_ses_idp; /* OUT: session id */ int lstio_ses_nmlen; /* IN: name length */ char *lstio_ses_namep; /* IN: session name */ @@ -242,6 +250,8 @@ typedef struct { typedef struct { lst_sid_t *lstio_ses_idp; /* OUT: session id */ int *lstio_ses_keyp; /* OUT: local key */ + /** OUT: session features */ + unsigned *lstio_ses_featp; lstcon_ndlist_ent_t *lstio_ses_ndinfo; /* OUT: */ int lstio_ses_nmlen; /* IN: name length */ char *lstio_ses_namep; /* OUT: session name */ @@ -303,6 +313,8 @@ typedef struct { int lstio_grp_nmlen; /* IN: name length */ char *lstio_grp_namep; /* IN: group name */ int lstio_grp_count; /* IN: # of nodes */ + /** OUT: session features */ + unsigned *lstio_grp_featp; lnet_process_id_t *lstio_grp_idsp; /* IN: nodes */ cfs_list_t *lstio_grp_resultp; /* OUT: list head of result buffer */ } lstio_group_nodes_args_t; diff --git a/lnet/selftest/brw_test.c b/lnet/selftest/brw_test.c index 345c828..e002e83 100644 --- a/lnet/selftest/brw_test.c +++ b/lnet/selftest/brw_test.c @@ -66,28 +66,54 @@ brw_client_fini (sfw_test_instance_t *tsi) int brw_client_init (sfw_test_instance_t *tsi) { - test_bulk_req_t *breq = &tsi->tsi_u.bulk; - int flags = breq->blk_flags; - int npg = breq->blk_npg; - srpc_bulk_t *bulk; - sfw_test_unit_t *tsu; - - LASSERT (tsi->tsi_is_client); - - if (npg > LNET_MAX_IOV || npg <= 0) - return -EINVAL; - - if (breq->blk_opc != LST_BRW_READ && breq->blk_opc != LST_BRW_WRITE) - return -EINVAL; - - if (flags != LST_BRW_CHECK_NONE && - flags != LST_BRW_CHECK_FULL && flags != LST_BRW_CHECK_SIMPLE) - return -EINVAL; + sfw_session_t *sn = tsi->tsi_batch->bat_session; + int flags; + int npg; + int len; + int opc; + srpc_bulk_t *bulk; + sfw_test_unit_t *tsu; + + LASSERT(sn != NULL); + LASSERT(tsi->tsi_is_client); + + if ((sn->sn_features & LST_FEAT_BULK_LEN) == 0) { + test_bulk_req_t *breq = &tsi->tsi_u.bulk_v0; + + opc = breq->blk_opc; + flags = breq->blk_flags; + npg = breq->blk_npg; + /* NB: this is not going to work for variable page size, + * but we have to keep it for compatibility */ + len = npg * CFS_PAGE_SIZE; + + } else { + test_bulk_req_v1_t *breq = &tsi->tsi_u.bulk_v1; + + /* I should never get this step if it's unknown feature + * because make_session will reject unknown feature */ + LASSERT((sn->sn_features & ~LST_FEATS_MASK) == 0); + + opc = breq->blk_opc; + flags = breq->blk_flags; + len = breq->blk_len; + npg = (len + CFS_PAGE_SIZE - 1) >> CFS_PAGE_SHIFT; + } + + if (npg > LNET_MAX_IOV || npg <= 0) + return -EINVAL; + + if (opc != LST_BRW_READ && opc != LST_BRW_WRITE) + return -EINVAL; + + if (flags != LST_BRW_CHECK_NONE && + flags != LST_BRW_CHECK_FULL && flags != LST_BRW_CHECK_SIMPLE) + return -EINVAL; cfs_list_for_each_entry_typed(tsu, &tsi->tsi_units, sfw_test_unit_t, tsu_list) { bulk = srpc_alloc_bulk(lnet_cpt_of_nid(tsu->tsu_dest.nid), - npg, breq->blk_opc == LST_BRW_READ); + npg, len, opc == LST_BRW_READ); if (bulk == NULL) { brw_client_fini(tsi); return -ENOMEM; @@ -96,7 +122,7 @@ brw_client_init (sfw_test_instance_t *tsi) tsu->tsu_private = bulk; } - return 0; + return 0; } #define BRW_POISON 0xbeefbeefbeefbeefULL @@ -237,32 +263,56 @@ brw_client_prep_rpc (sfw_test_unit_t *tsu, { srpc_bulk_t *bulk = tsu->tsu_private; sfw_test_instance_t *tsi = tsu->tsu_instance; - test_bulk_req_t *breq = &tsi->tsi_u.bulk; - int npg = breq->blk_npg; - int flags = breq->blk_flags; - srpc_client_rpc_t *rpc; - srpc_brw_reqst_t *req; - int rc; - - LASSERT (bulk != NULL); - LASSERT (bulk->bk_niov == npg); - - rc = sfw_create_test_rpc(tsu, dest, npg, npg * CFS_PAGE_SIZE, &rpc); - if (rc != 0) return rc; - - memcpy(&rpc->crpc_bulk, bulk, offsetof(srpc_bulk_t, bk_iovs[npg])); - if (breq->blk_opc == LST_BRW_WRITE) - brw_fill_bulk(&rpc->crpc_bulk, flags, BRW_MAGIC); - else - brw_fill_bulk(&rpc->crpc_bulk, flags, BRW_POISON); + sfw_session_t *sn = tsi->tsi_batch->bat_session; + srpc_client_rpc_t *rpc; + srpc_brw_reqst_t *req; + int flags; + int npg; + int len; + int opc; + int rc; + + LASSERT(sn != NULL); + LASSERT(bulk != NULL); + + if ((sn->sn_features & LST_FEAT_BULK_LEN) == 0) { + test_bulk_req_t *breq = &tsi->tsi_u.bulk_v0; + + opc = breq->blk_opc; + flags = breq->blk_flags; + npg = breq->blk_npg; + len = npg * CFS_PAGE_SIZE; + + } else { + test_bulk_req_v1_t *breq = &tsi->tsi_u.bulk_v1; + + /* I should never get this step if it's unknown feature + * because make_session will reject unknown feature */ + LASSERT((sn->sn_features & ~LST_FEATS_MASK) == 0); + + opc = breq->blk_opc; + flags = breq->blk_flags; + len = breq->blk_len; + npg = (len + CFS_PAGE_SIZE - 1) >> CFS_PAGE_SHIFT; + } + + rc = sfw_create_test_rpc(tsu, dest, sn->sn_features, npg, len, &rpc); + if (rc != 0) + return rc; - req = &rpc->crpc_reqstmsg.msg_body.brw_reqst; - req->brw_flags = flags; - req->brw_rw = breq->blk_opc; - req->brw_len = npg * CFS_PAGE_SIZE; + memcpy(&rpc->crpc_bulk, bulk, offsetof(srpc_bulk_t, bk_iovs[npg])); + if (opc == LST_BRW_WRITE) + brw_fill_bulk(&rpc->crpc_bulk, flags, BRW_MAGIC); + else + brw_fill_bulk(&rpc->crpc_bulk, flags, BRW_POISON); - *rpcpp = rpc; - return 0; + req = &rpc->crpc_reqstmsg.msg_body.brw_reqst; + req->brw_flags = flags; + req->brw_rw = opc; + req->brw_len = len; + + *rpcpp = rpc; + return 0; } static void @@ -379,6 +429,7 @@ brw_server_handle(struct srpc_server_rpc *rpc) srpc_msg_t *reqstmsg = &rpc->srpc_reqstbuf->buf_msg; srpc_brw_reply_t *reply = &replymsg->msg_body.brw_reply; srpc_brw_reqst_t *reqst = &reqstmsg->msg_body.brw_reqst; + int npg; int rc; LASSERT (sv->sv_id == SRPC_SERVICE_BRW); @@ -386,7 +437,6 @@ brw_server_handle(struct srpc_server_rpc *rpc) if (reqstmsg->msg_magic != SRPC_MSG_MAGIC) { LASSERT (reqstmsg->msg_magic == __swab32(SRPC_MSG_MAGIC)); - __swab32s(&reqstmsg->msg_type); __swab32s(&reqst->brw_rw); __swab32s(&reqst->brw_len); __swab32s(&reqst->brw_flags); @@ -395,11 +445,10 @@ brw_server_handle(struct srpc_server_rpc *rpc) } LASSERT (reqstmsg->msg_type == (__u32)srpc_service2request(sv->sv_id)); + reply->brw_status = 0; rpc->srpc_done = brw_server_rpc_done; if ((reqst->brw_rw != LST_BRW_READ && reqst->brw_rw != LST_BRW_WRITE) || - reqst->brw_len == 0 || (reqst->brw_len & ~CFS_PAGE_MASK) != 0 || - reqst->brw_len / CFS_PAGE_SIZE > LNET_MAX_IOV || (reqst->brw_flags != LST_BRW_CHECK_NONE && reqst->brw_flags != LST_BRW_CHECK_FULL && reqst->brw_flags != LST_BRW_CHECK_SIMPLE)) { @@ -407,10 +456,33 @@ brw_server_handle(struct srpc_server_rpc *rpc) return 0; } - reply->brw_status = 0; - /* allocate from "local" node */ - rc = sfw_alloc_pages(rpc, rpc->srpc_scd->scd_cpt, - reqst->brw_len / CFS_PAGE_SIZE, + if ((reqstmsg->msg_ses_feats & ~LST_FEATS_MASK) != 0) { + replymsg->msg_ses_feats = LST_FEATS_MASK; + reply->brw_status = EPROTO; + return 0; + } + + if ((reqstmsg->msg_ses_feats & LST_FEAT_BULK_LEN) == 0) { + /* compat with old version */ + if ((reqst->brw_len & ~CFS_PAGE_MASK) != 0) { + reply->brw_status = EINVAL; + return 0; + } + npg = reqst->brw_len >> CFS_PAGE_SHIFT; + + } else { + npg = (reqst->brw_len + CFS_PAGE_SIZE - 1) >> CFS_PAGE_SHIFT; + } + + replymsg->msg_ses_feats = reqstmsg->msg_ses_feats; + + if (reqst->brw_len == 0 || npg > LNET_MAX_IOV) { + reply->brw_status = EINVAL; + return 0; + } + + rc = sfw_alloc_pages(rpc, rpc->srpc_scd->scd_cpt, npg, + reqst->brw_len, reqst->brw_rw == LST_BRW_WRITE); if (rc != 0) return rc; diff --git a/lnet/selftest/conctl.c b/lnet/selftest/conctl.c index 901b6e2..9c2f39f 100644 --- a/lnet/selftest/conctl.c +++ b/lnet/selftest/conctl.c @@ -70,15 +70,15 @@ lst_session_new_ioctl(lstio_session_new_args_t *args) name[args->lstio_ses_nmlen] = 0; - rc = lstcon_session_new(name, - args->lstio_ses_key, - args->lstio_ses_timeout, - args->lstio_ses_force, - args->lstio_ses_idp); - - LIBCFS_FREE(name, args->lstio_ses_nmlen + 1); - - return rc; + rc = lstcon_session_new(name, + args->lstio_ses_key, + args->lstio_ses_feats, + args->lstio_ses_force, + args->lstio_ses_timeout, + args->lstio_ses_idp); + + LIBCFS_FREE(name, args->lstio_ses_nmlen + 1); + return rc; } int @@ -97,6 +97,7 @@ lst_session_info_ioctl(lstio_session_info_args_t *args) if (args->lstio_ses_idp == NULL || /* address for ouput sid */ args->lstio_ses_keyp == NULL || /* address for ouput key */ + args->lstio_ses_featp == NULL || /* address for ouput features */ args->lstio_ses_ndinfo == NULL || /* address for output ndinfo */ args->lstio_ses_namep == NULL || /* address for ouput name */ args->lstio_ses_nmlen <= 0 || @@ -105,6 +106,7 @@ lst_session_info_ioctl(lstio_session_info_args_t *args) return lstcon_session_info(args->lstio_ses_idp, args->lstio_ses_keyp, + args->lstio_ses_featp, args->lstio_ses_ndinfo, args->lstio_ses_namep, args->lstio_ses_nmlen); @@ -320,6 +322,7 @@ lst_group_update_ioctl(lstio_group_update_args_t *args) int lst_nodes_add_ioctl(lstio_group_nodes_args_t *args) { + unsigned feats; int rc; char *name; @@ -329,8 +332,9 @@ lst_nodes_add_ioctl(lstio_group_nodes_args_t *args) if (args->lstio_grp_idsp == NULL || /* array of ids */ args->lstio_grp_count <= 0 || args->lstio_grp_resultp == NULL || - args->lstio_grp_namep == NULL || - args->lstio_grp_nmlen <= 0 || + args->lstio_grp_featp == NULL || + args->lstio_grp_namep == NULL || + args->lstio_grp_nmlen <= 0 || args->lstio_grp_nmlen > LST_NAME_SIZE) return -EINVAL; @@ -348,10 +352,14 @@ lst_nodes_add_ioctl(lstio_group_nodes_args_t *args) name[args->lstio_grp_nmlen] = 0; rc = lstcon_nodes_add(name, args->lstio_grp_count, - args->lstio_grp_idsp, - args->lstio_grp_resultp); + args->lstio_grp_idsp, &feats, + args->lstio_grp_resultp); - LIBCFS_FREE(name, args->lstio_grp_nmlen + 1); + LIBCFS_FREE(name, args->lstio_grp_nmlen + 1); + if (rc == 0 && + cfs_copy_to_user(args->lstio_grp_featp, &feats, sizeof(feats))) { + return -EINVAL; + } return rc; } @@ -734,6 +742,12 @@ int lst_test_add_ioctl(lstio_test_args_t *args) args->lstio_tes_dgrp_nmlen > LST_NAME_SIZE) return -EINVAL; + if (args->lstio_tes_loop == 0 || /* negative is infinite */ + args->lstio_tes_concur <= 0 || + args->lstio_tes_dist <= 0 || + args->lstio_tes_span <= 0) + return -EINVAL; + /* have parameter, check if parameter length is valid */ if (args->lstio_tes_param != NULL && (args->lstio_tes_param_len <= 0 || diff --git a/lnet/selftest/conrpc.c b/lnet/selftest/conrpc.c index a4cc222..1213ef0 100644 --- a/lnet/selftest/conrpc.c +++ b/lnet/selftest/conrpc.c @@ -37,7 +37,7 @@ * * Console framework rpcs * - * Author: Liang Zhen + * Author: Liang Zhen */ #ifdef __KERNEL__ @@ -48,8 +48,8 @@ #include "conrpc.h" #include "console.h" -void lstcon_rpc_stat_reply(int, srpc_msg_t *, - lstcon_node_t *, lstcon_trans_stat_t *); +void lstcon_rpc_stat_reply(lstcon_rpc_trans_t *, srpc_msg_t *, + lstcon_node_t *, lstcon_trans_stat_t *); static void lstcon_rpc_done(srpc_client_rpc_t *rpc) @@ -90,13 +90,12 @@ lstcon_rpc_done(srpc_client_rpc_t *rpc) } int -lstcon_rpc_init(lstcon_node_t *nd, int service, - int npg, int cached, lstcon_rpc_t *crpc) +lstcon_rpc_init(lstcon_node_t *nd, int service, unsigned feats, + int bulk_npg, int bulk_len, int embedded, lstcon_rpc_t *crpc) { - - crpc->crp_rpc = sfw_create_rpc(nd->nd_id, service, - npg, npg * CFS_PAGE_SIZE, - lstcon_rpc_done, (void *)crpc); + crpc->crp_rpc = sfw_create_rpc(nd->nd_id, service, + feats, bulk_npg, bulk_len, + lstcon_rpc_done, (void *)crpc); if (crpc->crp_rpc == NULL) return -ENOMEM; @@ -107,7 +106,7 @@ lstcon_rpc_init(lstcon_node_t *nd, int service, crpc->crp_unpacked = 0; crpc->crp_status = 0; crpc->crp_stamp = 0; - crpc->crp_static = !cached; + crpc->crp_embedded = embedded; CFS_INIT_LIST_HEAD(&crpc->crp_link); cfs_atomic_inc(&console_session.ses_rpc_counter); @@ -116,8 +115,8 @@ lstcon_rpc_init(lstcon_node_t *nd, int service, } int -lstcon_rpc_prep(lstcon_node_t *nd, int service, - int npg, lstcon_rpc_t **crpcpp) +lstcon_rpc_prep(lstcon_node_t *nd, int service, unsigned feats, + int bulk_npg, int bulk_len, lstcon_rpc_t **crpcpp) { lstcon_rpc_t *crpc = NULL; int rc; @@ -138,7 +137,8 @@ lstcon_rpc_prep(lstcon_node_t *nd, int service, return -ENOMEM; } - rc = lstcon_rpc_init(nd, service, npg, 1, crpc); + rc = lstcon_rpc_init(nd, service, feats, + bulk_npg, bulk_len, 0, crpc); if (rc == 0) { *crpcpp = crpc; return 0; @@ -166,10 +166,10 @@ lstcon_rpc_put(lstcon_rpc_t *crpc) srpc_client_rpc_decref(crpc->crp_rpc); - if (crpc->crp_static) { - /* Static RPC, not allocated */ - memset(crpc, 0, sizeof(*crpc)); - crpc->crp_static = 1; + if (crpc->crp_embedded) { + /* embedded RPC, don't recycle it */ + memset(crpc, 0, sizeof(*crpc)); + crpc->crp_embedded = 1; } else { cfs_spin_lock(&console_session.ses_rpc_lock); @@ -256,10 +256,10 @@ lstcon_rpc_trans_prep(cfs_list_t *translist, LIBCFS_ALLOC(trans, sizeof(*trans)); if (trans == NULL) return -ENOMEM; - + trans->tas_opc = transop; - if (translist == NULL) + if (translist == NULL) CFS_INIT_LIST_HEAD(&trans->tas_olink); else cfs_list_add_tail(&trans->tas_olink, translist); @@ -270,9 +270,12 @@ lstcon_rpc_trans_prep(cfs_list_t *translist, cfs_atomic_set(&trans->tas_remaining, 0); cfs_waitq_init(&trans->tas_waitq); - *transpp = trans; + cfs_spin_lock(&console_session.ses_rpc_lock); + trans->tas_features = console_session.ses_features; + cfs_spin_unlock(&console_session.ses_rpc_lock); - return 0; + *transpp = trans; + return 0; } void @@ -406,7 +409,7 @@ lstcon_rpc_get_reply(lstcon_rpc_t *crpc, srpc_msg_t **msgpp) sfw_unpack_message(*msgpp); crpc->crp_unpacked = 1; } - + if (cfs_time_after(nd->nd_stamp, crpc->crp_stamp)) return 0; @@ -451,8 +454,12 @@ lstcon_rpc_trans_stat(lstcon_rpc_trans_t *trans, lstcon_trans_stat_t *stat) lstcon_rpc_stat_success(stat, 1); - lstcon_rpc_stat_reply(trans->tas_opc, rep, - crpc->crp_node, stat); + lstcon_rpc_stat_reply(trans, rep, crpc->crp_node, stat); + } + + if (trans->tas_opc == LST_TRANS_SESNEW && stat->trs_fwk_errno == 0) { + stat->trs_fwk_errno = + lstcon_session_feats_check(trans->tas_features); } CDEBUG(D_NET, "transaction %s : success %d, failure %d, total %d, " @@ -597,7 +604,8 @@ lstcon_rpc_trans_destroy(lstcon_rpc_trans_t *trans) } int -lstcon_sesrpc_prep(lstcon_node_t *nd, int transop, lstcon_rpc_t **crpc) +lstcon_sesrpc_prep(lstcon_node_t *nd, int transop, + unsigned feats, lstcon_rpc_t **crpc) { srpc_mksn_reqst_t *msrq; srpc_rmsn_reqst_t *rsrq; @@ -605,7 +613,8 @@ lstcon_sesrpc_prep(lstcon_node_t *nd, int transop, lstcon_rpc_t **crpc) switch (transop) { case LST_TRANS_SESNEW: - rc = lstcon_rpc_prep(nd, SRPC_SERVICE_MAKE_SESSION, 0, crpc); + rc = lstcon_rpc_prep(nd, SRPC_SERVICE_MAKE_SESSION, + feats, 0, 0, crpc); if (rc != 0) return rc; @@ -617,7 +626,8 @@ lstcon_sesrpc_prep(lstcon_node_t *nd, int transop, lstcon_rpc_t **crpc) break; case LST_TRANS_SESEND: - rc = lstcon_rpc_prep(nd, SRPC_SERVICE_REMOVE_SESSION, 0, crpc); + rc = lstcon_rpc_prep(nd, SRPC_SERVICE_REMOVE_SESSION, + feats, 0, 0, crpc); if (rc != 0) return rc; @@ -633,12 +643,12 @@ lstcon_sesrpc_prep(lstcon_node_t *nd, int transop, lstcon_rpc_t **crpc) } int -lstcon_dbgrpc_prep(lstcon_node_t *nd, lstcon_rpc_t **crpc) +lstcon_dbgrpc_prep(lstcon_node_t *nd, unsigned feats, lstcon_rpc_t **crpc) { - srpc_debug_reqst_t *drq; - int rc; + srpc_debug_reqst_t *drq; + int rc; - rc = lstcon_rpc_prep(nd, SRPC_SERVICE_DEBUG, 0, crpc); + rc = lstcon_rpc_prep(nd, SRPC_SERVICE_DEBUG, feats, 0, 0, crpc); if (rc != 0) return rc; @@ -646,19 +656,19 @@ lstcon_dbgrpc_prep(lstcon_node_t *nd, lstcon_rpc_t **crpc) drq->dbg_sid = console_session.ses_id; drq->dbg_flags = 0; - + return rc; } int -lstcon_batrpc_prep(lstcon_node_t *nd, int transop, - lstcon_tsb_hdr_t *tsb, lstcon_rpc_t **crpc) +lstcon_batrpc_prep(lstcon_node_t *nd, int transop, unsigned feats, + lstcon_tsb_hdr_t *tsb, lstcon_rpc_t **crpc) { - lstcon_batch_t *batch; - srpc_batch_reqst_t *brq; - int rc; + lstcon_batch_t *batch; + srpc_batch_reqst_t *brq; + int rc; - rc = lstcon_rpc_prep(nd, SRPC_SERVICE_BATCH, 0, crpc); + rc = lstcon_rpc_prep(nd, SRPC_SERVICE_BATCH, feats, 0, 0, crpc); if (rc != 0) return rc; @@ -679,17 +689,17 @@ lstcon_batrpc_prep(lstcon_node_t *nd, int transop, batch = (lstcon_batch_t *)tsb; brq->bar_arg = batch->bat_arg; - + return 0; } int -lstcon_statrpc_prep(lstcon_node_t *nd, lstcon_rpc_t **crpc) +lstcon_statrpc_prep(lstcon_node_t *nd, unsigned feats, lstcon_rpc_t **crpc) { - srpc_stat_reqst_t *srq; - int rc; + srpc_stat_reqst_t *srq; + int rc; - rc = lstcon_rpc_prep(nd, SRPC_SERVICE_QUERY_STAT, 0, crpc); + rc = lstcon_rpc_prep(nd, SRPC_SERVICE_QUERY_STAT, feats, 0, 0, crpc); if (rc != 0) return rc; @@ -776,7 +786,7 @@ int lstcon_pingrpc_prep(lst_test_ping_param_t *param, srpc_test_reqst_t *req) { test_ping_req_t *prq = &req->tsr_u.ping; - + prq->png_size = param->png_size; prq->png_flags = param->png_flags; /* TODO dest */ @@ -784,9 +794,9 @@ lstcon_pingrpc_prep(lst_test_ping_param_t *param, srpc_test_reqst_t *req) } int -lstcon_bulkrpc_prep(lst_test_bulk_param_t *param, srpc_test_reqst_t *req) +lstcon_bulkrpc_v0_prep(lst_test_bulk_param_t *param, srpc_test_reqst_t *req) { - test_bulk_req_t *brq = &req->tsr_u.bulk; + test_bulk_req_t *brq = &req->tsr_u.bulk_v0; brq->blk_opc = param->blk_opc; brq->blk_npg = (param->blk_size + CFS_PAGE_SIZE - 1) / CFS_PAGE_SIZE; @@ -796,7 +806,20 @@ lstcon_bulkrpc_prep(lst_test_bulk_param_t *param, srpc_test_reqst_t *req) } int -lstcon_testrpc_prep(lstcon_node_t *nd, int transop, +lstcon_bulkrpc_v1_prep(lst_test_bulk_param_t *param, srpc_test_reqst_t *req) +{ + test_bulk_req_v1_t *brq = &req->tsr_u.bulk_v1; + + brq->blk_opc = param->blk_opc; + brq->blk_flags = param->blk_flags; + brq->blk_len = param->blk_size; + brq->blk_offset = 0; /* reserved */ + + return 0; +} + +int +lstcon_testrpc_prep(lstcon_node_t *nd, int transop, unsigned feats, lstcon_test_t *test, lstcon_rpc_t **crpc) { lstcon_group_t *sgrp = test->tes_src_grp; @@ -804,14 +827,19 @@ lstcon_testrpc_prep(lstcon_node_t *nd, int transop, srpc_test_reqst_t *trq; srpc_bulk_t *bulk; int i; - int n = 0; - int rc = 0; - - if (transop == LST_TRANS_TSBCLIADD) - n = sfw_id_pages(test->tes_span); - - rc = lstcon_rpc_prep(nd, SRPC_SERVICE_TEST, n, crpc); - if (rc != 0) + int npg = 0; + int nob = 0; + int rc = 0; + + if (transop == LST_TRANS_TSBCLIADD) { + npg = sfw_id_pages(test->tes_span); + nob = (feats & LST_FEAT_BULK_LEN) == 0 ? + npg * CFS_PAGE_SIZE : + sizeof(lnet_process_id_packed_t) * test->tes_span; + } + + rc = lstcon_rpc_prep(nd, SRPC_SERVICE_TEST, feats, npg, nob, crpc); + if (rc != 0) return rc; trq = &(*crpc)->crp_rpc->crpc_reqstmsg.msg_body.tes_reqst; @@ -827,16 +855,24 @@ lstcon_testrpc_prep(lstcon_node_t *nd, int transop, } else { bulk = &(*crpc)->crp_rpc->crpc_bulk; - for (i = 0; i < n; i++) { - bulk->bk_iovs[i].kiov_offset = 0; - bulk->bk_iovs[i].kiov_len = CFS_PAGE_SIZE; - bulk->bk_iovs[i].kiov_page = cfs_alloc_page(CFS_ALLOC_STD); + for (i = 0; i < npg; i++) { + int len; - if (bulk->bk_iovs[i].kiov_page != NULL) - continue; + LASSERT(nob > 0); - lstcon_rpc_put(*crpc); - return -ENOMEM; + len = (feats & LST_FEAT_BULK_LEN) == 0 ? + CFS_PAGE_SIZE : min_t(int, nob, CFS_PAGE_SIZE); + nob -= len; + + bulk->bk_iovs[i].kiov_offset = 0; + bulk->bk_iovs[i].kiov_len = len; + bulk->bk_iovs[i].kiov_page = + cfs_alloc_page(CFS_ALLOC_STD); + + if (bulk->bk_iovs[i].kiov_page == NULL) { + lstcon_rpc_put(*crpc); + return -ENOMEM; + } } bulk->bk_sink = 0; @@ -844,8 +880,10 @@ lstcon_testrpc_prep(lstcon_node_t *nd, int transop, LASSERT (transop == LST_TRANS_TSBCLIADD); rc = lstcon_dstnodes_prep(test->tes_dst_grp, - test->tes_cliidx++, test->tes_dist, - test->tes_span, n, &bulk->bk_iovs[0]); + test->tes_cliidx++, + test->tes_dist, + test->tes_span, + npg, &bulk->bk_iovs[0]); if (rc != 0) { lstcon_rpc_put(*crpc); return rc; @@ -864,11 +902,20 @@ lstcon_testrpc_prep(lstcon_node_t *nd, int transop, switch (test->tes_type) { case LST_TEST_PING: trq->tsr_service = SRPC_SERVICE_PING; - rc = lstcon_pingrpc_prep((lst_test_ping_param_t *)&test->tes_param[0], trq); - break; - case LST_TEST_BULK: - trq->tsr_service = SRPC_SERVICE_BRW; - rc = lstcon_bulkrpc_prep((lst_test_bulk_param_t *)&test->tes_param[0], trq); + rc = lstcon_pingrpc_prep((lst_test_ping_param_t *) + &test->tes_param[0], trq); + break; + + case LST_TEST_BULK: + trq->tsr_service = SRPC_SERVICE_BRW; + if ((feats & LST_FEAT_BULK_LEN) == 0) { + rc = lstcon_bulkrpc_v0_prep((lst_test_bulk_param_t *) + &test->tes_param[0], trq); + } else { + rc = lstcon_bulkrpc_v1_prep((lst_test_bulk_param_t *) + &test->tes_param[0], trq); + } + break; default: LBUG(); @@ -878,11 +925,53 @@ lstcon_testrpc_prep(lstcon_node_t *nd, int transop, return rc; } +int +lstcon_sesnew_stat_reply(lstcon_rpc_trans_t *trans, + lstcon_node_t *nd, srpc_msg_t *reply) +{ + srpc_mksn_reply_t *mksn_rep = &reply->msg_body.mksn_reply; + int status = mksn_rep->mksn_status; + + if (status == 0 && + (reply->msg_ses_feats & ~LST_FEATS_MASK) != 0) { + mksn_rep->mksn_status = EPROTO; + status = EPROTO; + } + + if (status == EPROTO) { + CNETERR("session protocol error from %s: %u\n", + libcfs_nid2str(nd->nd_id.nid), + reply->msg_ses_feats); + } + + if (status != 0) + return status; + + if (!trans->tas_feats_updated) { + trans->tas_feats_updated = 1; + trans->tas_features = reply->msg_ses_feats; + } + + if (reply->msg_ses_feats != trans->tas_features) { + CNETERR("Framework features %x from %s is different with " + "features on this transaction: %x\n", + reply->msg_ses_feats, libcfs_nid2str(nd->nd_id.nid), + trans->tas_features); + status = mksn_rep->mksn_status = EPROTO; + } + + if (status == 0) { + /* session timeout on remote node */ + nd->nd_timeout = mksn_rep->mksn_timeout; + } + + return status; +} + void -lstcon_rpc_stat_reply(int transop, srpc_msg_t *msg, +lstcon_rpc_stat_reply(lstcon_rpc_trans_t *trans, srpc_msg_t *msg, lstcon_node_t *nd, lstcon_trans_stat_t *stat) { - srpc_mksn_reply_t *mksn_rep; srpc_rmsn_reply_t *rmsn_rep; srpc_debug_reply_t *dbg_rep; srpc_batch_reply_t *bat_rep; @@ -890,22 +979,15 @@ lstcon_rpc_stat_reply(int transop, srpc_msg_t *msg, srpc_stat_reply_t *stat_rep; int rc = 0; - switch (transop) { - case LST_TRANS_SESNEW: - mksn_rep = &msg->msg_body.mksn_reply; - - if (mksn_rep->mksn_status == 0) { + switch (trans->tas_opc) { + case LST_TRANS_SESNEW: + rc = lstcon_sesnew_stat_reply(trans, nd, msg); + if (rc == 0) { lstcon_sesop_stat_success(stat, 1); - /* session timeout on remote node */ - nd->nd_timeout = mksn_rep->mksn_timeout; return; } - LASSERT (mksn_rep->mksn_status == EBUSY || - mksn_rep->mksn_status == EINVAL); - lstcon_sesop_stat_failure(stat, 1); - rc = mksn_rep->mksn_status; break; case LST_TRANS_SESEND: @@ -917,9 +999,6 @@ lstcon_rpc_stat_reply(int transop, srpc_msg_t *msg, return; } - LASSERT (rmsn_rep->rmsn_status == EBUSY || - rmsn_rep->rmsn_status == EINVAL); - lstcon_sesop_stat_failure(stat, 1); rc = rmsn_rep->rmsn_status; break; @@ -931,9 +1010,7 @@ lstcon_rpc_stat_reply(int transop, srpc_msg_t *msg, if (dbg_rep->dbg_status == ESRCH) { lstcon_sesqry_stat_unknown(stat, 1); return; - } - - LASSERT (dbg_rep->dbg_status == 0); + } if (lstcon_session_match(dbg_rep->dbg_sid)) lstcon_sesqry_stat_active(stat, 1); @@ -951,7 +1028,7 @@ lstcon_rpc_stat_reply(int transop, srpc_msg_t *msg, } if (bat_rep->bar_status == EPERM && - transop == LST_TRANS_TSBSTOP) { + trans->tas_opc == LST_TRANS_TSBSTOP) { lstcon_tsbop_stat_success(stat, 1); return; } @@ -1021,6 +1098,7 @@ lstcon_rpc_trans_ndlist(cfs_list_t *ndlist, lstcon_ndlink_t *ndl; lstcon_node_t *nd; lstcon_rpc_t *rpc; + unsigned feats; int rc; /* Creating session RPG for list of nodes */ @@ -1031,6 +1109,7 @@ lstcon_rpc_trans_ndlist(cfs_list_t *ndlist, return rc; } + feats = trans->tas_features; cfs_list_for_each_entry_typed(ndl, ndlist, lstcon_ndlink_t, ndl_link) { rc = condition == NULL ? 1 : condition(transop, ndl->ndl_node, arg); @@ -1049,26 +1128,26 @@ lstcon_rpc_trans_ndlist(cfs_list_t *ndlist, switch (transop) { case LST_TRANS_SESNEW: case LST_TRANS_SESEND: - rc = lstcon_sesrpc_prep(nd, transop, &rpc); - break; - case LST_TRANS_SESQRY: - case LST_TRANS_SESPING: - rc = lstcon_dbgrpc_prep(nd, &rpc); - break; - case LST_TRANS_TSBCLIADD: - case LST_TRANS_TSBSRVADD: - rc = lstcon_testrpc_prep(nd, transop, - (lstcon_test_t *)arg, &rpc); - break; - case LST_TRANS_TSBRUN: - case LST_TRANS_TSBSTOP: - case LST_TRANS_TSBCLIQRY: - case LST_TRANS_TSBSRVQRY: - rc = lstcon_batrpc_prep(nd, transop, - (lstcon_tsb_hdr_t *)arg, &rpc); - break; - case LST_TRANS_STATQRY: - rc = lstcon_statrpc_prep(nd, &rpc); + rc = lstcon_sesrpc_prep(nd, transop, feats, &rpc); + break; + case LST_TRANS_SESQRY: + case LST_TRANS_SESPING: + rc = lstcon_dbgrpc_prep(nd, feats, &rpc); + break; + case LST_TRANS_TSBCLIADD: + case LST_TRANS_TSBSRVADD: + rc = lstcon_testrpc_prep(nd, transop, feats, + (lstcon_test_t *)arg, &rpc); + break; + case LST_TRANS_TSBRUN: + case LST_TRANS_TSBSTOP: + case LST_TRANS_TSBCLIQRY: + case LST_TRANS_TSBSRVQRY: + rc = lstcon_batrpc_prep(nd, transop, feats, + (lstcon_tsb_hdr_t *)arg, &rpc); + break; + case LST_TRANS_STATQRY: + rc = lstcon_statrpc_prep(nd, feats, &rpc); break; default: rc = -EINVAL; @@ -1080,7 +1159,7 @@ lstcon_rpc_trans_ndlist(cfs_list_t *ndlist, lstcon_rpc_trans_name(transop), rc); break; } - + lstcon_rpc_trans_addreq(trans, rpc); } @@ -1136,7 +1215,8 @@ lstcon_rpc_pinger(void *arg) if (nd->nd_state != LST_NODE_ACTIVE) continue; - rc = lstcon_sesrpc_prep(nd, LST_TRANS_SESEND, &crpc); + rc = lstcon_sesrpc_prep(nd, LST_TRANS_SESEND, + trans->tas_features, &crpc); if (rc != 0) { CERROR("Out of memory\n"); break; @@ -1169,7 +1249,7 @@ lstcon_rpc_pinger(void *arg) lstcon_rpc_get_reply(crpc, &rep); cfs_list_del_init(&crpc->crp_link); - + lstcon_rpc_put(crpc); } @@ -1181,7 +1261,8 @@ lstcon_rpc_pinger(void *arg) if (intv < (time_t)nd->nd_timeout / 2) continue; - rc = lstcon_rpc_init(nd, SRPC_SERVICE_DEBUG, 0, 0, crpc); + rc = lstcon_rpc_init(nd, SRPC_SERVICE_DEBUG, + trans->tas_features, 0, 0, 1, crpc); if (rc != 0) { CERROR("Out of memory\n"); break; diff --git a/lnet/selftest/conrpc.h b/lnet/selftest/conrpc.h index 52cbf11..503f481 100644 --- a/lnet/selftest/conrpc.h +++ b/lnet/selftest/conrpc.h @@ -26,6 +26,8 @@ /* * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. + * + * Copyright (c) 2012, Whamcloud Inc. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -35,7 +37,7 @@ * * Console rpc * - * Author: Liang Zhen + * Author: Liang Zhen */ #ifndef __LST_CONRPC_H__ @@ -71,7 +73,8 @@ typedef struct lstcon_rpc { int crp_posted:1; /* rpc is posted */ int crp_finished:1; /* rpc is finished */ int crp_unpacked:1; /* reply is unpacked */ - int crp_static:1; /* not from RPC buffer */ + /** RPC is embedded in other structure and can't free it */ + int crp_embedded:1; int crp_status; /* console rpc errors */ cfs_time_t crp_stamp; /* replied time stamp */ } lstcon_rpc_t; @@ -80,6 +83,10 @@ typedef struct lstcon_rpc_trans { cfs_list_t tas_olink; /* link chain on owner list */ cfs_list_t tas_link; /* link chain on global list */ int tas_opc; /* operation code of transaction */ + /* features mask is uptodate */ + unsigned tas_feats_updated; + /* test features mask */ + unsigned tas_features; cfs_waitq_t tas_waitq; /* wait queue head */ cfs_atomic_t tas_remaining; /* # of un-scheduled rpcs */ cfs_list_t tas_rpcs_list; /* queued requests */ @@ -104,14 +111,16 @@ typedef struct lstcon_rpc_trans { typedef int (* lstcon_rpc_cond_func_t)(int, struct lstcon_node *, void *); typedef int (* lstcon_rpc_readent_func_t)(int, srpc_msg_t *, lstcon_rpc_ent_t *); -int lstcon_sesrpc_prep(struct lstcon_node *nd, - int transop, lstcon_rpc_t **crpc); -int lstcon_dbgrpc_prep(struct lstcon_node *nd, lstcon_rpc_t **crpc); -int lstcon_batrpc_prep(struct lstcon_node *nd, int transop, +int lstcon_sesrpc_prep(struct lstcon_node *nd, int transop, + unsigned version, lstcon_rpc_t **crpc); +int lstcon_dbgrpc_prep(struct lstcon_node *nd, + unsigned version, lstcon_rpc_t **crpc); +int lstcon_batrpc_prep(struct lstcon_node *nd, int transop, unsigned version, struct lstcon_tsb_hdr *tsb, lstcon_rpc_t **crpc); -int lstcon_testrpc_prep(struct lstcon_node *nd, int transop, +int lstcon_testrpc_prep(struct lstcon_node *nd, int transop, unsigned version, struct lstcon_test *test, lstcon_rpc_t **crpc); -int lstcon_statrpc_prep(struct lstcon_node *nd, lstcon_rpc_t **crpc); +int lstcon_statrpc_prep(struct lstcon_node *nd, unsigned version, + lstcon_rpc_t **crpc); void lstcon_rpc_put(lstcon_rpc_t *crpc); int lstcon_rpc_trans_prep(cfs_list_t *translist, int transop, lstcon_rpc_trans_t **transpp); diff --git a/lnet/selftest/console.c b/lnet/selftest/console.c index 9dd1a49..2f4b543 100644 --- a/lnet/selftest/console.c +++ b/lnet/selftest/console.c @@ -406,9 +406,9 @@ lstcon_sesrpc_readent(int transop, srpc_msg_t *msg, } static int -lstcon_group_nodes_add(lstcon_group_t *grp, int count, - lnet_process_id_t *ids_up, - cfs_list_t *result_up) +lstcon_group_nodes_add(lstcon_group_t *grp, + int count, lnet_process_id_t *ids_up, + unsigned *featp, cfs_list_t *result_up) { lstcon_rpc_trans_t *trans; lstcon_ndlink_t *ndl; @@ -461,6 +461,8 @@ lstcon_group_nodes_add(lstcon_group_t *grp, int count, rc = lstcon_rpc_trans_interpreter(trans, result_up, lstcon_sesrpc_readent); + *featp = trans->tas_features; + /* destroy all RPGs */ lstcon_rpc_trans_destroy(trans); @@ -550,8 +552,8 @@ lstcon_group_add(char *name) } int -lstcon_nodes_add(char *name, int count, - lnet_process_id_t *ids_up, cfs_list_t *result_up) +lstcon_nodes_add(char *name, int count, lnet_process_id_t *ids_up, + unsigned *featp, cfs_list_t *result_up) { lstcon_group_t *grp; int rc; @@ -573,7 +575,7 @@ lstcon_nodes_add(char *name, int count, return -EBUSY; } - rc = lstcon_group_nodes_add(grp, count, ids_up, result_up); + rc = lstcon_group_nodes_add(grp, count, ids_up, featp, result_up); lstcon_group_put(grp); @@ -1265,7 +1267,7 @@ lstcon_test_add(char *name, int type, int loop, int concur, CDEBUG(D_NET, "Can't change running batch %s\n", name); return rc; } - + rc = lstcon_group_find(src_name, &src_grp); if (rc != 0) { CDEBUG(D_NET, "Can't find group %s\n", src_name); @@ -1680,8 +1682,8 @@ lstcon_new_session_id(lst_sid_t *sid) extern srpc_service_t lstcon_acceptor_service; int -lstcon_session_new(char *name, int key, - int timeout,int force, lst_sid_t *sid_up) +lstcon_session_new(char *name, int key, unsigned feats, + int timeout, int force, lst_sid_t *sid_up) { int rc = 0; int i; @@ -1689,8 +1691,8 @@ lstcon_session_new(char *name, int key, if (console_session.ses_state != LST_SESSION_NONE) { /* session exists */ if (!force) { - CERROR("Session %s already exists\n", - console_session.ses_name); + CNETERR("Session %s already exists\n", + console_session.ses_name); return -EEXIST; } @@ -1701,9 +1703,25 @@ lstcon_session_new(char *name, int key, return rc; } - for (i = 0; i < LST_GLOBAL_HASHSIZE; i++) { - LASSERT (cfs_list_empty(&console_session.ses_ndl_hash[i])); - } + if ((feats & ~LST_FEATS_MASK) != 0) { + CNETERR("Unknown session features %x\n", + (feats & ~LST_FEATS_MASK)); + return -EINVAL; + } + + for (i = 0; i < LST_GLOBAL_HASHSIZE; i++) + LASSERT(cfs_list_empty(&console_session.ses_ndl_hash[i])); + + lstcon_new_session_id(&console_session.ses_id); + + console_session.ses_key = key; + console_session.ses_state = LST_SESSION_ACTIVE; + console_session.ses_force = !!force; + console_session.ses_features = feats; + console_session.ses_feats_updated = 0; + console_session.ses_timeout = (timeout <= 0) ? + LST_CONSOLE_TIMEOUT : timeout; + strcpy(console_session.ses_name, name); rc = lstcon_batch_add(LST_DEFAULT_BATCH); if (rc != 0) @@ -1719,15 +1737,6 @@ lstcon_session_new(char *name, int key, return rc; } - lstcon_new_session_id(&console_session.ses_id); - - console_session.ses_key = key; - console_session.ses_state = LST_SESSION_ACTIVE; - console_session.ses_force = !!force; - console_session.ses_timeout = (timeout <= 0)? LST_CONSOLE_TIMEOUT: - timeout; - strcpy(console_session.ses_name, name); - if (cfs_copy_to_user(sid_up, &console_session.ses_id, sizeof(lst_sid_t)) == 0) return rc; @@ -1738,7 +1747,7 @@ lstcon_session_new(char *name, int key, } int -lstcon_session_info(lst_sid_t *sid_up, int *key_up, +lstcon_session_info(lst_sid_t *sid_up, int *key_up, unsigned *featp, lstcon_ndlist_ent_t *ndinfo_up, char *name_up, int len) { lstcon_ndlist_ent_t *entp; @@ -1760,7 +1769,10 @@ lstcon_session_info(lst_sid_t *sid_up, int *key_up, if (cfs_copy_to_user(sid_up, &console_session.ses_id, sizeof(lst_sid_t)) || - cfs_copy_to_user(key_up, &console_session.ses_key, sizeof(int)) || + cfs_copy_to_user(key_up, &console_session.ses_key, + sizeof(*key_up)) || + cfs_copy_to_user(featp, &console_session.ses_features, + sizeof(*featp)) || cfs_copy_to_user(ndinfo_up, entp, sizeof(*entp)) || cfs_copy_to_user(name_up, console_session.ses_name, len)) rc = -EFAULT; @@ -1804,6 +1816,7 @@ lstcon_session_end() console_session.ses_state = LST_SESSION_NONE; console_session.ses_key = 0; console_session.ses_force = 0; + console_session.ses_feats_updated = 0; /* destroy all batches */ while (!cfs_list_empty(&console_session.ses_bat_list)) { @@ -1831,6 +1844,38 @@ lstcon_session_end() return rc; } +int +lstcon_session_feats_check(unsigned feats) +{ + int rc = 0; + + if ((feats & ~LST_FEATS_MASK) != 0) { + CERROR("Can't support these features: %x\n", + (feats & ~LST_FEATS_MASK)); + return -EPROTO; + } + + cfs_spin_lock(&console_session.ses_rpc_lock); + + if (!console_session.ses_feats_updated) { + console_session.ses_feats_updated = 1; + console_session.ses_features = feats; + } + + if (console_session.ses_features != feats) + rc = -EPROTO; + + cfs_spin_unlock(&console_session.ses_rpc_lock); + + if (rc != 0) { + CERROR("remote features %x do not match with " + "session features %x of console\n", + feats, console_session.ses_features); + } + + return rc; +} + static int lstcon_acceptor_handle (srpc_server_rpc_t *rpc) { @@ -1853,6 +1898,11 @@ lstcon_acceptor_handle (srpc_server_rpc_t *rpc) goto out; } + if (lstcon_session_feats_check(req->msg_ses_feats) != 0) { + jrep->join_status = EPROTO; + goto out; + } + if (jreq->join_sid.ses_nid != LNET_NID_ANY && !lstcon_session_match(jreq->join_sid)) { jrep->join_status = EBUSY; @@ -1900,6 +1950,7 @@ lstcon_acceptor_handle (srpc_server_rpc_t *rpc) jrep->join_status = 0; out: + rep->msg_ses_feats = console_session.ses_features; if (grp != NULL) lstcon_group_put(grp); @@ -1931,12 +1982,14 @@ lstcon_console_init(void) memset(&console_session, 0, sizeof(lstcon_session_t)); - console_session.ses_id = LST_INVALID_SID; - console_session.ses_state = LST_SESSION_NONE; - console_session.ses_timeout = 0; - console_session.ses_force = 0; - console_session.ses_expired = 0; - console_session.ses_laststamp = cfs_time_current_sec(); + console_session.ses_id = LST_INVALID_SID; + console_session.ses_state = LST_SESSION_NONE; + console_session.ses_timeout = 0; + console_session.ses_force = 0; + console_session.ses_expired = 0; + console_session.ses_feats_updated = 0; + console_session.ses_features = LST_FEATS_MASK; + console_session.ses_laststamp = cfs_time_current_sec(); cfs_mutex_init(&console_session.ses_mutex); diff --git a/lnet/selftest/console.h b/lnet/selftest/console.h index 4e1ef96..6ea5ac5 100644 --- a/lnet/selftest/console.h +++ b/lnet/selftest/console.h @@ -138,9 +138,16 @@ typedef struct { int ses_state; /* state of session */ int ses_timeout; /* timeout in seconds */ time_t ses_laststamp; /* last operation stamp (seconds) */ - int ses_force:1; /* force creating */ - int ses_shutdown:1; /* session is shutting down */ - int ses_expired:1; /* console is timedout */ + /** tests features of the session */ + unsigned ses_features; + /** features are synced with remote test nodes */ + unsigned ses_feats_updated:1; + /** force creating */ + unsigned ses_force:1; + /** session is shutting down */ + unsigned ses_shutdown:1; + /** console is timedout */ + unsigned ses_expired:1; __u64 ses_id_cookie; /* batch id cookie */ char ses_name[LST_NAME_SIZE]; /* session name */ lstcon_rpc_trans_t *ses_ping; /* session pinger */ @@ -159,6 +166,7 @@ typedef struct { } lstcon_session_t; /*** session descriptor */ extern lstcon_session_t console_session; + static inline lstcon_trans_stat_t * lstcon_trans_stat(void) { @@ -174,13 +182,14 @@ lstcon_id2hash (lnet_process_id_t id, cfs_list_t *hash) } extern int lstcon_session_match(lst_sid_t sid); -extern int lstcon_session_new(char *name, int key, +extern int lstcon_session_new(char *name, int key, unsigned version, int timeout, int flags, lst_sid_t *sid_up); -extern int lstcon_session_info(lst_sid_t *sid_up, int *key, +extern int lstcon_session_info(lst_sid_t *sid_up, int *key, unsigned *verp, lstcon_ndlist_ent_t *entp, char *name_up, int len); extern int lstcon_session_end(void); extern int lstcon_session_debug(int timeout, cfs_list_t *result_up); -extern int lstcon_batch_debug(int timeout, char *name, +extern int lstcon_session_feats_check(unsigned feats); +extern int lstcon_batch_debug(int timeout, char *name, int client, cfs_list_t *result_up); extern int lstcon_group_debug(int timeout, char *name, cfs_list_t *result_up); @@ -191,7 +200,7 @@ extern int lstcon_group_del(char *name); extern int lstcon_group_clean(char *name, int args); extern int lstcon_group_refresh(char *name, cfs_list_t *result_up); extern int lstcon_nodes_add(char *name, int nnd, lnet_process_id_t *nds_up, - cfs_list_t *result_up); + unsigned *featp, cfs_list_t *result_up); extern int lstcon_nodes_remove(char *name, int nnd, lnet_process_id_t *nds_up, cfs_list_t *result_up); extern int lstcon_group_info(char *name, lstcon_ndlist_ent_t *gent_up, diff --git a/lnet/selftest/framework.c b/lnet/selftest/framework.c index 23bae6e..7d3e5af 100644 --- a/lnet/selftest/framework.c +++ b/lnet/selftest/framework.c @@ -276,7 +276,8 @@ sfw_session_expired (void *data) } static inline void -sfw_init_session (sfw_session_t *sn, lst_sid_t sid, const char *name) +sfw_init_session(sfw_session_t *sn, lst_sid_t sid, + unsigned features, const char *name) { stt_timer_t *timer = &sn->sn_timer; @@ -290,6 +291,7 @@ sfw_init_session (sfw_session_t *sn, lst_sid_t sid, const char *name) sn->sn_timer_active = 0; sn->sn_id = sid; + sn->sn_features = features; sn->sn_timeout = session_timeout; sn->sn_started = cfs_time_current(); @@ -432,9 +434,11 @@ sfw_get_stats (srpc_stat_reqst_t *request, srpc_stat_reply_t *reply) } int -sfw_make_session (srpc_mksn_reqst_t *request, srpc_mksn_reply_t *reply) +sfw_make_session(srpc_mksn_reqst_t *request, srpc_mksn_reply_t *reply) { - sfw_session_t *sn = sfw_data.fw_session; + sfw_session_t *sn = sfw_data.fw_session; + srpc_msg_t *msg = container_of(request, srpc_msg_t, + msg_body.mksn_reqst); if (request->mksn_sid.ses_nid == LNET_NID_ANY) { reply->mksn_sid = (sn == NULL) ? LST_INVALID_SID : sn->sn_id; @@ -459,6 +463,17 @@ sfw_make_session (srpc_mksn_reqst_t *request, srpc_mksn_reply_t *reply) } } + /* reject the request if it requires unknown features + * NB: old version will always accept all features because it's not + * aware of srpc_msg_t::msg_ses_feats, it's a defect but it's also + * harmless because it will return zero feature to console, and it's + * console's responsibility to make sure all nodes in a session have + * same feature mask. */ + if ((msg->msg_ses_feats & ~LST_FEATS_MASK) != 0) { + reply->mksn_status = EPROTO; + return 0; + } + /* brand new or create by force */ LIBCFS_ALLOC(sn, sizeof(sfw_session_t)); if (sn == NULL) { @@ -466,7 +481,8 @@ sfw_make_session (srpc_mksn_reqst_t *request, srpc_mksn_reply_t *reply) return -ENOMEM; } - sfw_init_session(sn, request->mksn_sid, &request->mksn_name[0]); + sfw_init_session(sn, request->mksn_sid, + msg->msg_ses_feats, &request->mksn_name[0]); cfs_spin_lock(&sfw_data.fw_lock); @@ -680,7 +696,7 @@ sfw_destroy_session (sfw_session_t *sn) } void -sfw_unpack_test_req (srpc_msg_t *msg) +sfw_unpack_addtest_req(srpc_msg_t *msg) { srpc_test_reqst_t *req = &msg->msg_body.tes_reqst; @@ -692,14 +708,25 @@ sfw_unpack_test_req (srpc_msg_t *msg) LASSERT (msg->msg_magic == __swab32(SRPC_MSG_MAGIC)); - if (req->tsr_service == SRPC_SERVICE_BRW) { - test_bulk_req_t *bulk = &req->tsr_u.bulk; + if (req->tsr_service == SRPC_SERVICE_BRW) { + if ((msg->msg_ses_feats & LST_FEAT_BULK_LEN) == 0) { + test_bulk_req_t *bulk = &req->tsr_u.bulk_v0; - __swab32s(&bulk->blk_opc); - __swab32s(&bulk->blk_npg); - __swab32s(&bulk->blk_flags); - return; - } + __swab32s(&bulk->blk_opc); + __swab32s(&bulk->blk_npg); + __swab32s(&bulk->blk_flags); + + } else { + test_bulk_req_v1_t *bulk = &req->tsr_u.bulk_v1; + + __swab16s(&bulk->blk_opc); + __swab16s(&bulk->blk_flags); + __swab32s(&bulk->blk_offset); + __swab32s(&bulk->blk_len); + } + + return; + } if (req->tsr_service == SRPC_SERVICE_PING) { test_ping_req_t *ping = &req->tsr_u.ping; @@ -766,9 +793,10 @@ sfw_add_test_instance (sfw_batch_t *tsb, srpc_server_rpc_t *rpc) LASSERT (bk->bk_pages != NULL); #endif LASSERT (bk->bk_niov * SFW_ID_PER_PAGE >= (unsigned int)ndest); - LASSERT ((unsigned int)bk->bk_len >= sizeof(lnet_process_id_t) * ndest); + LASSERT((unsigned int)bk->bk_len >= + sizeof(lnet_process_id_packed_t) * ndest); - sfw_unpack_test_req(msg); + sfw_unpack_addtest_req(msg); memcpy(&tsi->tsi_u, &req->tsr_u, sizeof(tsi->tsi_u)); for (i = 0; i < ndest; i++) { @@ -867,7 +895,7 @@ sfw_test_rpc_done (srpc_client_rpc_t *rpc) int done = 0; tsi->tsi_ops->tso_done_rpc(tsu, rpc); - + cfs_spin_lock(&tsi->tsi_lock); LASSERT (sfw_test_active(tsi)); @@ -896,8 +924,9 @@ sfw_test_rpc_done (srpc_client_rpc_t *rpc) } int -sfw_create_test_rpc (sfw_test_unit_t *tsu, lnet_process_id_t peer, - int nblk, int blklen, srpc_client_rpc_t **rpcpp) +sfw_create_test_rpc(sfw_test_unit_t *tsu, lnet_process_id_t peer, + unsigned features, int nblk, int blklen, + srpc_client_rpc_t **rpcpp) { srpc_client_rpc_t *rpc = NULL; sfw_test_instance_t *tsi = tsu->tsu_instance; @@ -926,13 +955,15 @@ sfw_create_test_rpc (sfw_test_unit_t *tsu, lnet_process_id_t peer, sfw_test_rpc_fini, tsu); } - if (rpc == NULL) { - CERROR ("Can't create rpc for test %d\n", tsi->tsi_service); - return -ENOMEM; - } + if (rpc == NULL) { + CERROR("Can't create rpc for test %d\n", tsi->tsi_service); + return -ENOMEM; + } - *rpcpp = rpc; - return 0; + rpc->crpc_reqstmsg.msg_ses_feats = features; + *rpcpp = rpc; + + return 0; } int @@ -1100,12 +1131,13 @@ sfw_free_pages (srpc_server_rpc_t *rpc) } int -sfw_alloc_pages(struct srpc_server_rpc *rpc, int cpt, int npages, int sink) +sfw_alloc_pages(struct srpc_server_rpc *rpc, int cpt, int npages, int len, + int sink) { LASSERT(rpc->srpc_bulk == NULL); LASSERT(npages > 0 && npages <= LNET_MAX_IOV); - rpc->srpc_bulk = srpc_alloc_bulk(cpt, npages, sink); + rpc->srpc_bulk = srpc_alloc_bulk(cpt, npages, len, sink); if (rpc->srpc_bulk == NULL) return -ENOMEM; @@ -1157,8 +1189,18 @@ sfw_add_test (srpc_server_rpc_t *rpc) if (request->tsr_is_client && rpc->srpc_bulk == NULL) { /* rpc will be resumed later in sfw_bulk_ready */ - return sfw_alloc_pages(rpc, CFS_CPT_ANY, - sfw_id_pages(request->tsr_ndest), 1); + int npg = sfw_id_pages(request->tsr_ndest); + int len; + + if ((sn->sn_features & LST_FEAT_BULK_LEN) == 0) { + len = npg * CFS_PAGE_SIZE; + + } else { + len = sizeof(lnet_process_id_packed_t) * + request->tsr_ndest; + } + + return sfw_alloc_pages(rpc, CFS_CPT_ANY, npg, len, 1); } rc = sfw_add_test_instance(bat, rpc); @@ -1217,9 +1259,10 @@ int sfw_handle_server_rpc(struct srpc_server_rpc *rpc) { struct srpc_service *sv = rpc->srpc_scd->scd_svc; - srpc_msg_t *reply = &rpc->srpc_replymsg; - srpc_msg_t *request = &rpc->srpc_reqstbuf->buf_msg; - int rc = 0; + srpc_msg_t *reply = &rpc->srpc_replymsg; + srpc_msg_t *request = &rpc->srpc_reqstbuf->buf_msg; + unsigned features = LST_FEATS_MASK; + int rc = 0; LASSERT (sfw_data.fw_active_srpc == NULL); LASSERT (sv->sv_id <= SRPC_FRAMEWORK_SERVICE_MAX_ID); @@ -1245,6 +1288,31 @@ sfw_handle_server_rpc(struct srpc_server_rpc *rpc) sfw_unpack_message(request); LASSERT (request->msg_type == srpc_service2request(sv->sv_id)); + /* rpc module should have checked this */ + LASSERT(request->msg_version == SRPC_MSG_VERSION); + + if (sv->sv_id != SRPC_SERVICE_MAKE_SESSION && + sv->sv_id != SRPC_SERVICE_DEBUG) { + sfw_session_t *sn = sfw_data.fw_session; + + if (sn != NULL && + sn->sn_features != request->msg_ses_feats) { + CNETERR("Features of framework RPC don't match " + "features of current session: %x/%x\n", + request->msg_ses_feats, sn->sn_features); + reply->msg_body.reply.status = EPROTO; + reply->msg_body.reply.sid = sn->sn_id; + goto out; + } + + } else if ((request->msg_ses_feats & ~LST_FEATS_MASK) != 0) { + /* NB: at this point, old version will ignore features and + * create new session anyway, so console should be able + * to handle this */ + reply->msg_body.reply.status = EPROTO; + goto out; + } + switch(sv->sv_id) { default: LBUG (); @@ -1278,6 +1346,10 @@ sfw_handle_server_rpc(struct srpc_server_rpc *rpc) break; } + if (sfw_data.fw_session != NULL) + features = sfw_data.fw_session->sn_features; + out: + reply->msg_ses_feats = features; rpc->srpc_done = sfw_server_rpc_done; cfs_spin_lock(&sfw_data.fw_lock); @@ -1348,11 +1420,11 @@ sfw_bulk_ready(struct srpc_server_rpc *rpc, int status) } srpc_client_rpc_t * -sfw_create_rpc (lnet_process_id_t peer, int service, - int nbulkiov, int bulklen, - void (*done) (srpc_client_rpc_t *), void *priv) +sfw_create_rpc(lnet_process_id_t peer, int service, + unsigned features, int nbulkiov, int bulklen, + void (*done)(srpc_client_rpc_t *), void *priv) { - srpc_client_rpc_t *rpc; + srpc_client_rpc_t *rpc = NULL; cfs_spin_lock(&sfw_data.fw_lock); @@ -1363,19 +1435,25 @@ sfw_create_rpc (lnet_process_id_t peer, int service, rpc = cfs_list_entry(sfw_data.fw_zombie_rpcs.next, srpc_client_rpc_t, crpc_list); cfs_list_del(&rpc->crpc_list); - cfs_spin_unlock(&sfw_data.fw_lock); srpc_init_client_rpc(rpc, peer, service, 0, 0, done, sfw_client_rpc_fini, priv); - return rpc; } cfs_spin_unlock(&sfw_data.fw_lock); - rpc = srpc_create_client_rpc(peer, service, nbulkiov, bulklen, done, - nbulkiov != 0 ? NULL : sfw_client_rpc_fini, - priv); - return rpc; + if (rpc == NULL) { + rpc = srpc_create_client_rpc(peer, service, + nbulkiov, bulklen, done, + nbulkiov != 0 ? NULL : + sfw_client_rpc_fini, + priv); + } + + if (rpc != NULL) /* "session" is concept in framework */ + rpc->crpc_reqstmsg.msg_ses_feats = features; + + return rpc; } void @@ -1384,10 +1462,9 @@ sfw_unpack_message (srpc_msg_t *msg) if (msg->msg_magic == SRPC_MSG_MAGIC) return; /* no flipping needed */ + /* srpc module should guarantee I wouldn't get crap */ LASSERT (msg->msg_magic == __swab32(SRPC_MSG_MAGIC)); - __swab32s(&msg->msg_type); - if (msg->msg_type == SRPC_MSG_STAT_REQST) { srpc_stat_reqst_t *req = &msg->msg_body.stat_reqst; diff --git a/lnet/selftest/ping_test.c b/lnet/selftest/ping_test.c index 6fa4c55..8a1ddd3 100644 --- a/lnet/selftest/ping_test.c +++ b/lnet/selftest/ping_test.c @@ -55,7 +55,10 @@ static lst_ping_data_t lst_ping_data; static int ping_client_init(sfw_test_instance_t *tsi) { - LASSERT (tsi->tsi_is_client); + sfw_session_t *sn = tsi->tsi_batch->bat_session; + + LASSERT(tsi->tsi_is_client); + LASSERT(sn != NULL && (sn->sn_features & ~LST_FEATS_MASK) == 0); cfs_spin_lock_init(&lst_ping_data.pnd_lock); lst_ping_data.pnd_counter = 0; @@ -81,13 +84,18 @@ ping_client_fini (sfw_test_instance_t *tsi) static int ping_client_prep_rpc(sfw_test_unit_t *tsu, - lnet_process_id_t dest, srpc_client_rpc_t **rpc) + lnet_process_id_t dest, srpc_client_rpc_t **rpc) { - srpc_ping_reqst_t *req; - struct timeval tv; - int rc; + srpc_ping_reqst_t *req; + sfw_test_instance_t *tsi = tsu->tsu_instance; + sfw_session_t *sn = tsi->tsi_batch->bat_session; + struct timeval tv; + int rc; - rc = sfw_create_test_rpc(tsu, dest, 0, 0, rpc); + LASSERT(sn != NULL); + LASSERT((sn->sn_features & ~LST_FEATS_MASK) == 0); + + rc = sfw_create_test_rpc(tsu, dest, sn->sn_features, 0, 0, rpc); if (rc != 0) return rc; @@ -124,14 +132,14 @@ ping_client_done_rpc (sfw_test_unit_t *tsu, srpc_client_rpc_t *rpc) libcfs_id2str(rpc->crpc_dest), reqst->pnr_seq, rpc->crpc_status); return; - } + } if (rpc->crpc_replymsg.msg_magic != SRPC_MSG_MAGIC) { __swab32s(&reply->pnr_seq); __swab32s(&reply->pnr_magic); __swab32s(&reply->pnr_status); } - + if (reply->pnr_magic != LST_PING_TEST_MAGIC) { rpc->crpc_status = -EBADMSG; cfs_atomic_inc(&sn->sn_ping_errors); @@ -139,8 +147,8 @@ ping_client_done_rpc (sfw_test_unit_t *tsu, srpc_client_rpc_t *rpc) reply->pnr_magic, libcfs_id2str(rpc->crpc_dest), LST_PING_TEST_MAGIC); return; - } - + } + if (reply->pnr_seq != reqst->pnr_seq) { rpc->crpc_status = -EBADMSG; cfs_atomic_inc(&sn->sn_ping_errors); @@ -162,6 +170,7 @@ ping_server_handle(struct srpc_server_rpc *rpc) { struct srpc_service *sv = rpc->srpc_scd->scd_svc; srpc_msg_t *reqstmsg = &rpc->srpc_reqstbuf->buf_msg; + srpc_msg_t *replymsg = &rpc->srpc_replymsg; srpc_ping_reqst_t *req = &reqstmsg->msg_body.ping_reqst; srpc_ping_reply_t *rep = &rpc->srpc_replymsg.msg_body.ping_reply; @@ -170,7 +179,6 @@ ping_server_handle(struct srpc_server_rpc *rpc) if (reqstmsg->msg_magic != SRPC_MSG_MAGIC) { LASSERT (reqstmsg->msg_magic == __swab32(SRPC_MSG_MAGIC)); - __swab32s(&reqstmsg->msg_type); __swab32s(&req->pnr_seq); __swab32s(&req->pnr_magic); __swab64s(&req->pnr_time_sec); @@ -187,9 +195,17 @@ ping_server_handle(struct srpc_server_rpc *rpc) rep->pnr_seq = req->pnr_seq; rep->pnr_magic = LST_PING_TEST_MAGIC; - CDEBUG (D_NET, "Get ping %d from %s\n", - req->pnr_seq, libcfs_id2str(rpc->srpc_peer)); - return 0; + if ((reqstmsg->msg_ses_feats & ~LST_FEATS_MASK) != 0) { + replymsg->msg_ses_feats = LST_FEATS_MASK; + rep->pnr_status = EPROTO; + return 0; + } + + replymsg->msg_ses_feats = reqstmsg->msg_ses_feats; + + CDEBUG(D_NET, "Get ping %d from %s\n", + req->pnr_seq, libcfs_id2str(rpc->srpc_peer)); + return 0; } sfw_test_client_ops_t ping_test_client; diff --git a/lnet/selftest/rpc.c b/lnet/selftest/rpc.c index 2193a3a..20358fe 100644 --- a/lnet/selftest/rpc.c +++ b/lnet/selftest/rpc.c @@ -85,23 +85,26 @@ void srpc_set_counters (const srpc_counters_t *cnt) cfs_spin_unlock(&srpc_data.rpc_glock); } -void -srpc_add_bulk_page (srpc_bulk_t *bk, cfs_page_t *pg, int i) +int +srpc_add_bulk_page(srpc_bulk_t *bk, cfs_page_t *pg, int i, int nob) { - LASSERT (i >= 0 && i < bk->bk_niov); + nob = min(nob, (int)CFS_PAGE_SIZE); + + LASSERT(nob > 0); + LASSERT(i >= 0 && i < bk->bk_niov); #ifdef __KERNEL__ - bk->bk_iovs[i].kiov_offset = 0; - bk->bk_iovs[i].kiov_page = pg; - bk->bk_iovs[i].kiov_len = CFS_PAGE_SIZE; + bk->bk_iovs[i].kiov_offset = 0; + bk->bk_iovs[i].kiov_page = pg; + bk->bk_iovs[i].kiov_len = nob; #else - LASSERT (bk->bk_pages != NULL); + LASSERT(bk->bk_pages != NULL); - bk->bk_pages[i] = pg; - bk->bk_iovs[i].iov_len = CFS_PAGE_SIZE; - bk->bk_iovs[i].iov_base = cfs_page_address(pg); + bk->bk_pages[i] = pg; + bk->bk_iovs[i].iov_len = nob; + bk->bk_iovs[i].iov_base = cfs_page_address(pg); #endif - return; + return nob; } void @@ -134,54 +137,56 @@ srpc_free_bulk (srpc_bulk_t *bk) } srpc_bulk_t * -srpc_alloc_bulk(int cpt, int npages, int sink) +srpc_alloc_bulk(int cpt, unsigned bulk_npg, unsigned bulk_len, int sink) { - srpc_bulk_t *bk; - cfs_page_t **pages; - int i; + srpc_bulk_t *bk; + cfs_page_t **pages; + int i; - LASSERT (npages > 0 && npages <= LNET_MAX_IOV); + LASSERT(bulk_npg > 0 && bulk_npg <= LNET_MAX_IOV); LIBCFS_CPT_ALLOC(bk, lnet_cpt_table(), cpt, - offsetof(srpc_bulk_t, bk_iovs[npages])); - if (bk == NULL) { - CERROR ("Can't allocate descriptor for %d pages\n", npages); - return NULL; - } + offsetof(srpc_bulk_t, bk_iovs[bulk_npg])); + if (bk == NULL) { + CERROR("Can't allocate descriptor for %d pages\n", bulk_npg); + return NULL; + } - memset(bk, 0, offsetof(srpc_bulk_t, bk_iovs[npages])); - bk->bk_sink = sink; - bk->bk_niov = npages; - bk->bk_len = npages * CFS_PAGE_SIZE; + memset(bk, 0, offsetof(srpc_bulk_t, bk_iovs[bulk_npg])); + bk->bk_sink = sink; + bk->bk_len = bulk_len; + bk->bk_niov = bulk_npg; #ifndef __KERNEL__ LIBCFS_CPT_ALLOC(pages, lnet_cpt_table(), cpt, - sizeof(cfs_page_t *) * npages); - if (pages == NULL) { - LIBCFS_FREE(bk, offsetof(srpc_bulk_t, bk_iovs[npages])); - CERROR ("Can't allocate page array for %d pages\n", npages); - return NULL; - } + sizeof(cfs_page_t *) * bulk_npg); + if (pages == NULL) { + LIBCFS_FREE(bk, offsetof(srpc_bulk_t, bk_iovs[bulk_npg])); + CERROR("Can't allocate page array for %d pages\n", bulk_npg); + return NULL; + } - memset(pages, 0, sizeof(cfs_page_t *) * npages); - bk->bk_pages = pages; + memset(pages, 0, sizeof(cfs_page_t *) * bulk_npg); + bk->bk_pages = pages; #else - UNUSED (pages); + UNUSED(pages); #endif - for (i = 0; i < npages; i++) { + for (i = 0; i < bulk_npg; i++) { cfs_page_t *pg; + int nob; pg = cfs_page_cpt_alloc(lnet_cpt_table(), cpt, CFS_ALLOC_STD); - if (pg == NULL) { - CERROR ("Can't allocate page %d of %d\n", i, npages); - srpc_free_bulk(bk); - return NULL; - } + if (pg == NULL) { + CERROR("Can't allocate page %d of %d\n", i, bulk_npg); + srpc_free_bulk(bk); + return NULL; + } - srpc_add_bulk_page(bk, pg, i); - } + nob = srpc_add_bulk_page(bk, pg, i, bulk_len); + bulk_len -= nob; + } - return bk; + return bk; } static inline __u64 @@ -1026,22 +1031,25 @@ srpc_handle_rpc(swi_workitem_t *wi) if (msg->msg_magic == 0) { /* moaned already in srpc_lnet_ev_handler */ - rc = EBADMSG; - } else if (msg->msg_version != SRPC_MSG_VERSION && - msg->msg_version != __swab32(SRPC_MSG_VERSION)) { - CWARN ("Version mismatch: %u, %u expected, from %s\n", - msg->msg_version, SRPC_MSG_VERSION, - libcfs_id2str(rpc->srpc_peer)); - reply->status = EPROTO; - } else { - reply->status = 0; - rc = (*sv->sv_handler) (rpc); - LASSERT (reply->status == 0 || !rpc->srpc_bulk); - } + srpc_server_rpc_done(rpc, EBADMSG); + return 1; + } - if (rc != 0) { - srpc_server_rpc_done(rpc, rc); - return 1; + srpc_unpack_msg_hdr(msg); + if (msg->msg_version != SRPC_MSG_VERSION) { + CWARN("Version mismatch: %u, %u expected, from %s\n", + msg->msg_version, SRPC_MSG_VERSION, + libcfs_id2str(rpc->srpc_peer)); + reply->status = EPROTO; + /* drop through and send reply */ + } else { + reply->status = 0; + rc = (*sv->sv_handler)(rpc); + LASSERT(reply->status == 0 || !rpc->srpc_bulk); + if (rc != 0) { + srpc_server_rpc_done(rpc, rc); + return 1; + } } wi->swi_state = SWI_STATE_BULK_STARTED; @@ -1257,10 +1265,9 @@ srpc_send_rpc (swi_workitem_t *wi) rc = rpc->crpc_replyev.ev_status; if (rc != 0) break; - if ((reply->msg_type != type && - reply->msg_type != __swab32(type)) || - (reply->msg_magic != SRPC_MSG_MAGIC && - reply->msg_magic != __swab32(SRPC_MSG_MAGIC))) { + srpc_unpack_msg_hdr(reply); + if (reply->msg_type != type || + reply->msg_magic != SRPC_MSG_MAGIC) { CWARN ("Bad message from %s: type %u (%d expected)," " magic %u (%d expected).\n", libcfs_id2str(rpc->crpc_dest), @@ -1363,7 +1370,6 @@ srpc_post_rpc (srpc_client_rpc_t *rpc) { LASSERT (!rpc->crpc_aborted); LASSERT (srpc_data.rpc_state == SRPC_STATE_RUNNING); - LASSERT ((rpc->crpc_bulk.bk_len & ~CFS_PAGE_MASK) == 0); CDEBUG (D_NET, "Posting RPC: peer %s, service %d, timeout %d\n", libcfs_id2str(rpc->crpc_dest), rpc->crpc_service, diff --git a/lnet/selftest/rpc.h b/lnet/selftest/rpc.h index 505c0b5..25a37d0 100644 --- a/lnet/selftest/rpc.h +++ b/lnet/selftest/rpc.h @@ -175,32 +175,44 @@ typedef struct { } WIRE_ATTR test_bulk_req_t; typedef struct { - __u32 png_size; /* size of ping message */ - __u32 png_flags; /* reserved flags */ + /** bulk operation code */ + __u16 blk_opc; + /** data check flags */ + __u16 blk_flags; + /** data length */ + __u32 blk_len; + /** reserved: offset */ + __u32 blk_offset; +} WIRE_ATTR test_bulk_req_v1_t; + +typedef struct { + __u32 png_size; /* size of ping message */ + __u32 png_flags; /* reserved flags */ } WIRE_ATTR test_ping_req_t; typedef struct { - __u64 tsr_rpyid; /* reply buffer matchbits */ - __u64 tsr_bulkid; /* bulk buffer matchbits */ - lst_sid_t tsr_sid; /* session id */ - lst_bid_t tsr_bid; /* batch id */ - __u32 tsr_service; /* test type: bulk|ping|... */ - /* test client loop count or # server buffers needed */ - __u32 tsr_loop; - __u32 tsr_concur; /* concurrency of test */ - __u8 tsr_is_client; /* is test client or not */ - __u8 tsr_stop_onerr; /* stop on error */ - __u32 tsr_ndest; /* # of dest nodes */ + __u64 tsr_rpyid; /* reply buffer matchbits */ + __u64 tsr_bulkid; /* bulk buffer matchbits */ + lst_sid_t tsr_sid; /* session id */ + lst_bid_t tsr_bid; /* batch id */ + __u32 tsr_service; /* test type: bulk|ping|... */ + /* test client loop count or # server buffers needed */ + __u32 tsr_loop; + __u32 tsr_concur; /* concurrency of test */ + __u8 tsr_is_client; /* is test client or not */ + __u8 tsr_stop_onerr; /* stop on error */ + __u32 tsr_ndest; /* # of dest nodes */ union { - test_bulk_req_t bulk; - test_ping_req_t ping; - } tsr_u; + test_ping_req_t ping; + test_bulk_req_t bulk_v0; + test_bulk_req_v1_t bulk_v1; + } tsr_u; } WIRE_ATTR srpc_test_reqst_t; typedef struct { - __u32 tsr_status; /* returned code */ - lst_sid_t tsr_sid; + __u32 tsr_status; /* returned code */ + lst_sid_t tsr_sid; } WIRE_ATTR srpc_test_reply_t; /* TEST RPCs */ @@ -232,14 +244,18 @@ typedef struct { #define SRPC_MSG_MAGIC 0xeeb0f00d #define SRPC_MSG_VERSION 1 + typedef struct srpc_msg { - __u32 msg_magic; /* magic */ - __u32 msg_version; /* # version */ - /* what's in msg_body? srpc_msg_type_t */ + /** magic number */ + __u32 msg_magic; + /** message version number */ + __u32 msg_version; + /** type of message body: srpc_msg_type_t */ __u32 msg_type; - __u32 msg_reserved0; /* reserved seats */ - __u32 msg_reserved1; - __u32 msg_reserved2; + __u32 msg_reserved0; + __u32 msg_reserved1; + /** test session features */ + __u32 msg_ses_feats; union { srpc_generic_reqst_t reqst; srpc_generic_reply_t reply; @@ -268,4 +284,18 @@ typedef struct srpc_msg { #include +static inline void +srpc_unpack_msg_hdr(srpc_msg_t *msg) +{ + if (msg->msg_magic == SRPC_MSG_MAGIC) + return; /* no flipping needed */ + + __swab32s(&msg->msg_magic); + __swab32s(&msg->msg_type); + __swab32s(&msg->msg_version); + __swab32s(&msg->msg_ses_feats); + __swab32s(&msg->msg_reserved0); + __swab32s(&msg->msg_reserved1); +} + #endif /* __SELFTEST_RPC_H__ */ diff --git a/lnet/selftest/selftest.h b/lnet/selftest/selftest.h index 786501d..76a6ecf 100644 --- a/lnet/selftest/selftest.h +++ b/lnet/selftest/selftest.h @@ -338,6 +338,7 @@ typedef struct { lst_sid_t sn_id; /* unique identifier */ unsigned int sn_timeout; /* # seconds' inactivity to expire */ int sn_timer_active; + unsigned int sn_features; stt_timer_t sn_timer; cfs_list_t sn_batches; /* list of batches */ char sn_name[LST_NAME_SIZE]; @@ -389,10 +390,11 @@ typedef struct sfw_test_instance { cfs_list_t tsi_free_rpcs; /* free rpcs */ cfs_list_t tsi_active_rpcs; /* active rpcs */ - union { - test_bulk_req_t bulk; /* bulk parameter */ - test_ping_req_t ping; /* ping parameter */ - } tsi_u; + union { + test_ping_req_t ping; /* ping parameter */ + test_bulk_req_t bulk_v0; /* bulk parameter */ + test_bulk_req_v1_t bulk_v1; /* bulk v1 parameter */ + } tsi_u; } sfw_test_instance_t; /* XXX: trailing (CFS_PAGE_SIZE % sizeof(lnet_process_id_t)) bytes at @@ -418,17 +420,20 @@ typedef struct sfw_test_case { } sfw_test_case_t; srpc_client_rpc_t * -sfw_create_rpc(lnet_process_id_t peer, int service, int nbulkiov, int bulklen, - void (*done) (srpc_client_rpc_t *), void *priv); -int sfw_create_test_rpc(sfw_test_unit_t *tsu, lnet_process_id_t peer, - int nblk, int blklen, srpc_client_rpc_t **rpc); +sfw_create_rpc(lnet_process_id_t peer, int service, + unsigned features, int nbulkiov, int bulklen, + void (*done) (srpc_client_rpc_t *), void *priv); +int sfw_create_test_rpc(sfw_test_unit_t *tsu, + lnet_process_id_t peer, unsigned features, + int nblk, int blklen, srpc_client_rpc_t **rpc); void sfw_abort_rpc(srpc_client_rpc_t *rpc); void sfw_post_rpc(srpc_client_rpc_t *rpc); void sfw_client_rpc_done(srpc_client_rpc_t *rpc); void sfw_unpack_message(srpc_msg_t *msg); void sfw_free_pages(srpc_server_rpc_t *rpc); void sfw_add_bulk_page(srpc_bulk_t *bk, cfs_page_t *pg, int i); -int sfw_alloc_pages(srpc_server_rpc_t *rpc, int cpt, int npages, int sink); +int sfw_alloc_pages(srpc_server_rpc_t *rpc, int cpt, int npages, int len, + int sink); int sfw_make_session (srpc_mksn_reqst_t *request, srpc_mksn_reply_t *reply); srpc_client_rpc_t * @@ -439,7 +444,8 @@ srpc_create_client_rpc(lnet_process_id_t peer, int service, void srpc_post_rpc(srpc_client_rpc_t *rpc); void srpc_abort_rpc(srpc_client_rpc_t *rpc, int why); void srpc_free_bulk(srpc_bulk_t *bk); -srpc_bulk_t *srpc_alloc_bulk(int cpt, int npages, int sink); +srpc_bulk_t *srpc_alloc_bulk(int cpt, unsigned bulk_npg, unsigned bulk_len, + int sink); int srpc_send_rpc(swi_workitem_t *wi); int srpc_send_reply(srpc_server_rpc_t *rpc); int srpc_add_service(srpc_service_t *sv); diff --git a/lnet/utils/lst.c b/lnet/utils/lst.c index ee835f1..4f40bcd 100644 --- a/lnet/utils/lst.c +++ b/lnet/utils/lst.c @@ -41,11 +41,21 @@ #include #include #include - +/* NB: these includes are layering violation */ +#include +#include lst_sid_t LST_INVALID_SID = {LNET_NID_ANY, -1}; static lst_sid_t session_id; static int session_key; + +#if LUSTRE_VERSION_CODE >= OBD_OCD_VERSION(2, 6, 50, 0) +/* assume all nodes can understand feature LST_FEAT_BULK_LEN */ +static unsigned session_features = LST_FEATS_MASK; +#else +static unsigned session_features = LST_FEATS_EMPTY; +#endif + static lstcon_trans_stat_t trans_stat; typedef struct list_string { @@ -454,7 +464,7 @@ lst_print_transerr(cfs_list_t *head, char *optstr) continue; } - fprintf(stderr, "%s failed on %s: %s\n", + fprintf(stderr, "operation %s failed on %s: %s\n", optstr, libcfs_id2str(ent->rpe_peer), strerror(ent->rpe_fwk_errno)); } @@ -503,7 +513,7 @@ lst_ioctl(unsigned int opc, void *buf, int len) } int -lst_new_session_ioctl (char *name, int timeout, int force, lst_sid_t *sid) +lst_new_session_ioctl(char *name, int timeout, int force, lst_sid_t *sid) { lstio_session_new_args_t args = {0}; @@ -511,6 +521,7 @@ lst_new_session_ioctl (char *name, int timeout, int force, lst_sid_t *sid) args.lstio_ses_timeout = timeout; args.lstio_ses_force = force; args.lstio_ses_idp = sid; + args.lstio_ses_feats = session_features; args.lstio_ses_nmlen = strlen(name); args.lstio_ses_namep = name; @@ -598,32 +609,31 @@ jt_lst_new_session(int argc, char **argv) } rc = lst_new_session_ioctl(name, timeout, force, &session_id); - if (rc != 0) { lst_print_error("session", "Failed to create session: %s\n", strerror(errno)); return rc; } - fprintf(stdout, "SESSION: %s TIMEOUT: %d FORCE: %s\n", - name, timeout, force ? "Yes": "No"); - - return rc; + fprintf(stdout, "SESSION: %s FEATURES: %x TIMEOUT: %d FORCE: %s\n", + name, session_features, timeout, force ? "Yes" : "No"); + return 0; } int -lst_session_info_ioctl(char *name, int len, int *key, - lst_sid_t *sid, lstcon_ndlist_ent_t *ndinfo) +lst_session_info_ioctl(char *name, int len, int *key, unsigned *featp, + lst_sid_t *sid, lstcon_ndlist_ent_t *ndinfo) { - lstio_session_info_args_t args = {0}; + lstio_session_info_args_t args = {0}; - args.lstio_ses_idp = sid; - args.lstio_ses_keyp = key; - args.lstio_ses_ndinfo = ndinfo; - args.lstio_ses_nmlen = len; - args.lstio_ses_namep = name; + args.lstio_ses_idp = sid; + args.lstio_ses_keyp = key; + args.lstio_ses_featp = featp; + args.lstio_ses_ndinfo = ndinfo; + args.lstio_ses_nmlen = len; + args.lstio_ses_namep = name; - return lst_ioctl(LSTIO_SESSION_INFO, &args, sizeof(args)); + return lst_ioctl(LSTIO_SESSION_INFO, &args, sizeof(args)); } int @@ -632,10 +642,12 @@ jt_lst_show_session(int argc, char **argv) lstcon_ndlist_ent_t ndinfo; lst_sid_t sid; char name[LST_NAME_SIZE]; - int key; - int rc; + unsigned feats; + int key; + int rc; - rc = lst_session_info_ioctl(name, LST_NAME_SIZE, &key, &sid, &ndinfo); + rc = lst_session_info_ioctl(name, LST_NAME_SIZE, &key, + &feats, &sid, &ndinfo); if (rc != 0) { lst_print_error("session", "Failed to show session: %s\n", @@ -643,9 +655,9 @@ jt_lst_show_session(int argc, char **argv) return -1; } - fprintf(stdout, "%s ID: "LPU64"@%s, KEY: %d NODES: %d\n", - name, sid.ses_stamp, libcfs_nid2str(sid.ses_nid), - key, ndinfo.nle_nnode); + fprintf(stdout, "%s ID: "LPU64"@%s, KEY: %d FEATURES: %x NODES: %d\n", + name, sid.ses_stamp, libcfs_nid2str(sid.ses_nid), + key, feats, ndinfo.nle_nnode); return 0; } @@ -726,14 +738,15 @@ lst_get_node_count(int type, char *str, int *countp, lnet_process_id_t **idspp) lstcon_test_batch_ent_t ent; lstcon_ndlist_ent_t *entp = &ent.tbe_cli_nle; lst_sid_t sid; - int key; - int rc; + unsigned feats; + int key; + int rc; - switch (type) { - case LST_OPC_SESSION: - rc = lst_session_info_ioctl(buf, LST_NAME_SIZE, - &key, &sid, entp); - break; + switch (type) { + case LST_OPC_SESSION: + rc = lst_session_info_ioctl(buf, LST_NAME_SIZE, + &key, &feats, &sid, entp); + break; case LST_OPC_BATCHSRV: entp = &ent.tbe_srv_nle; @@ -896,7 +909,7 @@ out: int lst_add_nodes_ioctl (char *name, int count, lnet_process_id_t *ids, - cfs_list_t *resultp) + unsigned *featp, cfs_list_t *resultp) { lstio_group_nodes_args_t args = {0}; @@ -904,6 +917,7 @@ lst_add_nodes_ioctl (char *name, int count, lnet_process_id_t *ids, args.lstio_grp_nmlen = strlen(name); args.lstio_grp_namep = name; args.lstio_grp_count = count; + args.lstio_grp_featp = featp; args.lstio_grp_idsp = ids; args.lstio_grp_resultp = resultp; @@ -928,6 +942,7 @@ jt_lst_add_group(int argc, char **argv) cfs_list_t head; lnet_process_id_t *ids; char *name; + unsigned feats = session_features; int count; int rc; int i; @@ -974,31 +989,50 @@ jt_lst_add_group(int argc, char **argv) rc = lst_alloc_rpcent(&head, count, 0); if (rc != 0) { fprintf(stderr, "Out of memory\n"); - break; - } + return -1; + } - rc = lst_add_nodes_ioctl(name, count, ids, &head); + rc = lst_add_nodes_ioctl(name, count, ids, &feats, &head); - free(ids); + free(ids); - if (rc == 0) { - lst_free_rpcent(&head); - fprintf(stderr, "%s are added to session\n", argv[i]); - continue; - } + if (rc != 0) + goto failed; - if (rc == -1) { - lst_free_rpcent(&head); - lst_print_error("group", "Failed to add nodes %s: %s\n", - argv[i], strerror(errno)); - break; - } + fprintf(stdout, "%s are added to session\n", argv[i]); - lst_print_transerr(&head, "create session"); - lst_free_rpcent(&head); - } + if ((feats & session_features) != session_features) { + fprintf(stdout, + "Warning, this session will run with " + "compatible mode because some test nodes " + "might not understand these features: %x\n", + (~feats & session_features)); + } - return rc; + lst_free_rpcent(&head); + } + + return 0; + +failed: + if (rc == -1) { + lst_print_error("group", "Failed to add nodes %s: %s\n", + argv[i], strerror(errno)); + + } else { + if (trans_stat.trs_fwk_errno == EPROTO) { + fprintf(stderr, + "test nodes might have different LST " + "features, please disable some features by " + "setting LST_FEATURES\n"); + } + + lst_print_transerr(&head, "create session"); + } + + lst_free_rpcent(&head); + + return rc; } int @@ -3194,7 +3228,20 @@ static command_t lst_cmdlist[] = { int lst_initialize(void) { - char *key; + char *key; + char *feats; + + feats = getenv("LST_FEATURES"); + if (feats != NULL) + session_features = strtol(feats, NULL, 16); + + if ((session_features & ~LST_FEATS_MASK) != 0) { + fprintf(stderr, + "Unsupported session features %x, " + "only support these features so far: %x\n", + (session_features & ~LST_FEATS_MASK), LST_FEATS_MASK); + return -1; + } key = getenv("LST_SESSION"); diff --git a/lnet/utils/lstclient.c b/lnet/utils/lstclient.c index d79d343..e440b41 100644 --- a/lnet/utils/lstclient.c +++ b/lnet/utils/lstclient.c @@ -56,6 +56,7 @@ static struct option lstjn_options[] = { {"sesid", required_argument, 0, 's' }, {"group", required_argument, 0, 'g' }, + {"features", required_argument, 0, 'f' }, {"server_mode", no_argument, 0, 'm' }, {0, 0, 0, 0 } }; @@ -74,7 +75,7 @@ lstjn_rpc_done(srpc_client_rpc_t *rpc) } int -lstjn_join_session(char *ses, char *grp) +lstjn_join_session(char *ses, char *grp, unsigned feats) { lnet_process_id_t sesid; srpc_client_rpc_t *rpc; @@ -91,8 +92,8 @@ lstjn_join_session(char *ses, char *grp) return -1; } - rpc = sfw_create_rpc(sesid, SRPC_SERVICE_JOIN, 0, - 0, lstjn_rpc_done, NULL); + rpc = sfw_create_rpc(sesid, SRPC_SERVICE_JOIN, feats, + 0, 0, lstjn_rpc_done, NULL); if (rpc == NULL) { fprintf(stderr, "Out of memory\n"); return -1; @@ -129,6 +130,17 @@ lstjn_join_session(char *ses, char *grp) return -1; } + if (rpc->crpc_replymsg.msg_ses_feats != feats) { + /* this can only happen when connecting to old console + * which will ignore features */ + fprintf(stderr, "Can't join session %s group %s because " + "feature bits can't match: %x/%x, please set " + "feature bits by -f FEATURES and retry\n", + ses, grp, feats, rpc->crpc_replymsg.msg_ses_feats); + srpc_client_rpc_decref(rpc); + return -1; + } + sreq = &rpc->crpc_reqstmsg.msg_body.mksn_reqst; sreq->mksn_sid = rep->join_sid; sreq->mksn_force = 0; @@ -156,19 +168,21 @@ lstjn_join_session(char *ses, char *grp) int main(int argc, char **argv) { - char *ses = NULL; - char *grp = NULL; - int server_mode_flag = 0; - int optidx; - int c; - int rc; - - const char *usage_string = - "Usage: lstclient --sesid ID --group GROUP [--server_mode]\n"; - - while (1) { - c = getopt_long(argc, argv, "s:g:m", - lstjn_options, &optidx); + char *ses = NULL; + char *grp = NULL; + unsigned feats = LST_FEATS_MASK; + int server_mode_flag = 0; + int optidx; + int c; + int rc; + + const char *usage_string = + "Usage: lstclient --sesid ID --group GROUP " + "--features FEATURES [--server_mode]\n"; + + while (1) { + c = getopt_long(argc, argv, "s:g:f:m", + lstjn_options, &optidx); if (c == -1) break; @@ -180,6 +194,10 @@ main(int argc, char **argv) case 'g': grp = optarg; break; + case 'f': + feats = strtol(optarg, NULL, 16); + break; + case 'm': server_mode_flag = 1; break; @@ -194,26 +212,33 @@ main(int argc, char **argv) return -1; } - rc = libcfs_debug_init(5 * 1024 * 1024); - if (rc != 0) { - CERROR("libcfs_debug_init() failed: %d\n", rc); - return -1; - } - - rc = cfs_wi_startup(); - if (rc != 0) { - CERROR("cfs_wi_startup() failed: %d\n", rc); - libcfs_debug_cleanup(); - return -1; - } - - rc = LNetInit(); - if (rc != 0) { - CERROR("LNetInit() failed: %d\n", rc); - cfs_wi_shutdown(); - libcfs_debug_cleanup(); - return -1; - } + if ((feats & ~LST_FEATS_MASK) != 0) { + fprintf(stderr, + "lstclient can't understand these feature bits: %x\n", + (feats & ~LST_FEATS_MASK)); + return -1; + } + + rc = libcfs_debug_init(5 * 1024 * 1024); + if (rc != 0) { + fprintf(stderr, "libcfs_debug_init() failed: %d\n", rc); + return -1; + } + + rc = cfs_wi_startup(); + if (rc != 0) { + fprintf(stderr, "cfs_wi_startup() failed: %d\n", rc); + libcfs_debug_cleanup(); + return -1; + } + + rc = LNetInit(); + if (rc != 0) { + fprintf(stderr, "LNetInit() failed: %d\n", rc); + cfs_wi_shutdown(); + libcfs_debug_cleanup(); + return -1; + } if (server_mode_flag) lnet_server_mode(); @@ -227,7 +252,7 @@ main(int argc, char **argv) return -1; } - rc = lstjn_join_session(ses, grp); + rc = lstjn_join_session(ses, grp, feats); if (rc != 0) goto out; -- 1.8.3.1